{"meta":{"date":"2026-09-20","timezone":"UTC","generated_at":"2026-10-14T23:17:09Z","environments":["dev","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":36,"failed_runs":30,"runs_with_occurrences":30,"runs_with_failure_pattern_matches":27,"runs_with_unmatched_occurrences":3,"runs_without_occurrence_rows":6,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7032,"pr_state":"closed","pr_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","final_merged_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T23:42:35Z","started_at":"2026-09-20T23:42:35Z","completed_at":"2026-09-21T02:00:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T02:12:59.831769704Z","timing_metadata_checked_at":"2026-09-21T02:12:59.831769704Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T02:12:59.841473272Z","region_metadata_checked_at":"2026-09-21T02:12:59.841473272Z"},"failed_at":["alert","e2e"],"failed_test_count":17,"occurrences":[{"row_id":"02e9809cd7f8d23e2c737a00661471038d58db90188e40af0036237302688400","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"0a72c2fe9bb203f971f92d8e51c30ced5c7cc31b87aab0f0b5be2aa7c9918b6a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa01c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-rsgvv4vlx279\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"name\": \"f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:58.238504906Z\",\n \"endTime\": \"2026-09-21T00:56:02.680818108Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa01c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-rsgvv4vlx279\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"name\": \"f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:58.238504906Z\",\n \"endTime\": \"2026-09-21T00:56:02.680818108Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0b82c1db6950a781a267177aa64950fde6f6c8c501e5777f20ac6dc4e992b4b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"e8bc9a8486674bdc1a81049ad931ccbb5a9f98bfa53795eca763544cc96cde0e","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001366090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-fxwzs2\" in resourcegroup=\"rg-np-downgrade-fxwzs2-xgl2x967w267\" to finish deleting: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"name\": \"bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:01:37.424253631Z\",\n \"endTime\": \"2026-09-21T01:25:46.248853963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2nnff2dqdtbc8d6g85073gq2fnh75 still exists (deletion dispatched at 2026-09-21T01:01:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001366090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-fxwzs2\" in resourcegroup=\"rg-np-downgrade-fxwzs2-xgl2x967w267\" to finish deleting: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"name\": \"bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:01:37.424253631Z\",\n \"endTime\": \"2026-09-21T01:25:46.248853963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2nnff2dqdtbc8d6g85073gq2fnh75 still exists (deletion dispatched at 2026-09-21T01:01:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"13653feef8e0f6a8d62ace6219529068afa54cbf24b52723dad793923f9d0346","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"alert","signature_id":"5c8b752be3d2e2575305bb4467922ed388fb9c7274d746c67ce437e73fc6372a","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:59:00Z\n Ended: 2026-09-21T01:07:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2niah7ifd1s9jttm16f840o7s2bba\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2niah7ifd1s9jttm16f840o7s2bba/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T01:01:00Z\n Ended: 2026-09-21T01:07:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T01:02:03Z\n Ended: 2026-09-21T01:09:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T01:02:03Z\n Ended: 2026-09-21T01:09:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"2fc7f3ee9f94dbc32ffede670008c32d38d69a6aef96224a8d7e994f64e149c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"1d7e40ff9457b0a0fd1f510836392b2a61ea6d2ec70f11c154614ed442a91215","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006001c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nzxl7wns7x4f\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"name\": \"5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:09.310854698Z\",\n \"endTime\": \"2026-09-21T00:56:12.748107548Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006001c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nzxl7wns7x4f\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"name\": \"5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:09.310854698Z\",\n \"endTime\": \"2026-09-21T00:56:12.748107548Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b05405f46dafc8dd0ae920633fddb44c6abaf21393a6366db687ccaa33eed04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"8e12d0852cea0a6ef4c2f62124f08fedad1faab413533a1887ee7ffe0186f31d","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00079e860\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-rn5wh2crpv9m\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeede74-90f9-493d-a157-ba91fc8a2853\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeede74-90f9-493d-a157-ba91fc8a2853\",\n \"name\": \"afeede74-90f9-493d-a157-ba91fc8a2853\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:08.330592373Z\",\n \"endTime\": \"2026-09-21T00:56:12.711427367Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00079e860\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-rn5wh2crpv9m\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeede74-90f9-493d-a157-ba91fc8a2853\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeede74-90f9-493d-a157-ba91fc8a2853\",\n \"name\": \"afeede74-90f9-493d-a157-ba91fc8a2853\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:08.330592373Z\",\n \"endTime\": \"2026-09-21T00:56:12.711427367Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3e354663fb55940cef665b7bddd1207955cf524117dee53dcd495c5fe945cba8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"alert","signature_id":"911f13dc127a9ace065557f3d9d6b1c823b667662d3dabe8b6a218f33ca1bf47","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 24 time(s) (12 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:36:05Z\n Ended: 2026-09-21T01:14:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:37:08Z\n Ended: 2026-09-21T01:15:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T00:37:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-21T00:40:06Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-21T00:40:06Z\n Ended: 2026-09-21T01:15:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-21T00:40:06Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-21T00:41:05Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-21T00:41:05Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"41f70c76d62be6ed3c95c79b5bcb4deb2205122d96e4de3070ef772aa2f3d4d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"36da18cbc5bd0fcbf1bcfacf064639d80f3a1a5cf27962716b02e77a1bbfec85","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001226220\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ctqghmzsdxw5\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"name\": \"ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:10.309010209Z\",\n \"endTime\": \"2026-09-21T00:56:12.764905068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001226220\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ctqghmzsdxw5\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"name\": \"ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:10.309010209Z\",\n \"endTime\": \"2026-09-21T00:56:12.764905068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"583638edb772369086d3b0e1b57352af5a237f8b1c86615613d108d1eb374a5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"d6906ab33d96cae1ef1353787751cc0c7d0fd1b9e2f1ba9fdecffdd4391cd20a","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d521c0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-drf4xvsgws57\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\",\n \"name\": \"2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.910293207Z\",\n \"endTime\": \"2026-09-21T00:56:22.909745001Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d521c0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-drf4xvsgws57\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\",\n \"name\": \"2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.910293207Z\",\n \"endTime\": \"2026-09-21T00:56:22.909745001Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"63ce4156fcf1566568f31ab427bbf46d4e0aef0fb149ba8a81baf762a89792e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"5f4073b27d81b6ba979875f303e0026e5d88b19010e94d8fbda6cdd43405762d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-q2fxxd\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00161c340\u003e: \n failed to create HCP cluster cluster-zstream-5-0-q2fxxd: failed waiting for cluster=\"cluster-zstream-5-0-q2fxxd\" in resourcegroup=\"rg-zstream-upgrade-5-0-66rntshp9zkw\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfab690-d284-4689-86d1-d3a143d2d2a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfab690-d284-4689-86d1-d3a143d2d2a4\",\n \"name\": \"bcfab690-d284-4689-86d1-d3a143d2d2a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:48.427657091Z\",\n \"endTime\": \"2026-09-21T00:55:52.803861648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-q2fxxd\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00161c340\u003e: \n failed to create HCP cluster cluster-zstream-5-0-q2fxxd: failed waiting for cluster=\"cluster-zstream-5-0-q2fxxd\" in resourcegroup=\"rg-zstream-upgrade-5-0-66rntshp9zkw\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfab690-d284-4689-86d1-d3a143d2d2a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfab690-d284-4689-86d1-d3a143d2d2a4\",\n \"name\": \"bcfab690-d284-4689-86d1-d3a143d2d2a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:48.427657091Z\",\n \"endTime\": \"2026-09-21T00:55:52.803861648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"66302ecd8bbb01fe9e6cde62c84746691d393007bc9ba37a80f94662326a4338","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"fb228348ca1cd58b436b3544af73f3d69eabd197936f9379767b0fcf28b2b9e2","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bcc520\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-qxqh2rljxqb9\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81fd376-89d4-498f-a01a-21c1ae71f7f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81fd376-89d4-498f-a01a-21c1ae71f7f3\",\n \"name\": \"e81fd376-89d4-498f-a01a-21c1ae71f7f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.989793193Z\",\n \"endTime\": \"2026-09-21T00:56:22.708890844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bcc520\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-qxqh2rljxqb9\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81fd376-89d4-498f-a01a-21c1ae71f7f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81fd376-89d4-498f-a01a-21c1ae71f7f3\",\n \"name\": \"e81fd376-89d4-498f-a01a-21c1ae71f7f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.989793193Z\",\n \"endTime\": \"2026-09-21T00:56:22.708890844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6bda78852f729ba5db83309e5ae946c855ff8cb7239afd63b88b7f10983c11ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"alert","signature_id":"59cd894a17dcad23b6224f622452e17a2a68039a078585dd58b1284a023c6bf0","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 24 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:46:08Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:47:08Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T00:47:09Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T00:47:09Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-21T00:47:09Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-21T00:48:11Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-21T00:48:11Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-21T00:48:11Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:16:10Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-21T00:51:12Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-21T00:51:13Z\n Ended: 2026-09-21T01:17:13Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-21T00:51:13Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-21T00:55:14Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"7d1053a7dacf27d1191a5b4c50a6f4a65a1870b33f294a87ef00cbc2c1a7e56b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"3e0062883f6a7b1aa6861d60c4689b05572197b0ccbb7b68cf2a6bb313f310ed","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7eba0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-h6stp5d8n7w6\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17c59b53-7724-4c26-9b50-9bd1833df54f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17c59b53-7724-4c26-9b50-9bd1833df54f\",\n \"name\": \"17c59b53-7724-4c26-9b50-9bd1833df54f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:54.807177783Z\",\n \"endTime\": \"2026-09-21T00:56:02.769493408Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7eba0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-h6stp5d8n7w6\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17c59b53-7724-4c26-9b50-9bd1833df54f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17c59b53-7724-4c26-9b50-9bd1833df54f\",\n \"name\": \"17c59b53-7724-4c26-9b50-9bd1833df54f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:54.807177783Z\",\n \"endTime\": \"2026-09-21T00:56:02.769493408Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8e3d20099ff5b5be552409ff2bfe189304c5bb37544e083866e6e0e592b743cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"e126abbe3fd20bf154b2a5a83d9a532f73d8c15de2b02bdae2fe166758e207a1","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0e120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-wq4gdf4lv86c\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6274357-fa39-41af-8e56-d9301737abf7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6274357-fa39-41af-8e56-d9301737abf7\",\n \"name\": \"c6274357-fa39-41af-8e56-d9301737abf7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:48.844013158Z\",\n \"endTime\": \"2026-09-21T00:55:52.749315715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0e120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-wq4gdf4lv86c\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6274357-fa39-41af-8e56-d9301737abf7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6274357-fa39-41af-8e56-d9301737abf7\",\n \"name\": \"c6274357-fa39-41af-8e56-d9301737abf7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:48.844013158Z\",\n \"endTime\": \"2026-09-21T00:55:52.749315715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c1ccb14bbc156214eee6c0ac0814461afde9dd1b3342907b465dda023ed9cf65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"7423340e196d990fa25ef185f5f19b2603dd00ed381a561dc6f7f6a62639d096","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001798260\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6v4jlrzkkq4h\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40607b22-0990-4b1a-a924-bff4ba978495\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40607b22-0990-4b1a-a924-bff4ba978495\",\n \"name\": \"40607b22-0990-4b1a-a924-bff4ba978495\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:11.931014232Z\",\n \"endTime\": \"2026-09-21T00:56:12.766689996Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001798260\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6v4jlrzkkq4h\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40607b22-0990-4b1a-a924-bff4ba978495\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40607b22-0990-4b1a-a924-bff4ba978495\",\n \"name\": \"40607b22-0990-4b1a-a924-bff4ba978495\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:11.931014232Z\",\n \"endTime\": \"2026-09-21T00:56:12.766689996Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dece95162694e4a26ef44b56ca69cfa85e5d2c029ff18c92cd562c510863c221","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"86240fd1d6cc9a1cfec477b60417613aecc1fa5308bed9e0b396aa7862dc467a","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b963c0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d6mjtfggws4p\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc3c40a7-208d-4f13-910b-8b0045d5faab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc3c40a7-208d-4f13-910b-8b0045d5faab\",\n \"name\": \"bc3c40a7-208d-4f13-910b-8b0045d5faab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:14.850025669Z\",\n \"endTime\": \"2026-09-21T00:56:22.569668641Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b963c0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d6mjtfggws4p\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc3c40a7-208d-4f13-910b-8b0045d5faab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc3c40a7-208d-4f13-910b-8b0045d5faab\",\n \"name\": \"bc3c40a7-208d-4f13-910b-8b0045d5faab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:14.850025669Z\",\n \"endTime\": \"2026-09-21T00:56:22.569668641Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e58f63cd83b40490fb19357f0377e98c86a800a074eb36daeeb6227654b79964","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"4566340774f4d5644a077677c8965bc9db5a33855b406b923890c6903dd9241c","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00082c560\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-rd9wvhphr7cj\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f761987-c863-4a01-80d2-bd3d7356613c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f761987-c863-4a01-80d2-bd3d7356613c\",\n \"name\": \"7f761987-c863-4a01-80d2-bd3d7356613c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:29:35.145071349Z\",\n \"endTime\": \"2026-09-21T00:58:42.745453014Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00082c560\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-rd9wvhphr7cj\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f761987-c863-4a01-80d2-bd3d7356613c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f761987-c863-4a01-80d2-bd3d7356613c\",\n \"name\": \"7f761987-c863-4a01-80d2-bd3d7356613c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:29:35.145071349Z\",\n \"endTime\": \"2026-09-21T00:58:42.745453014Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f2af2b309eacd7d0408fd20356814a6464f3b98ff772ef35253a4563c84ee280","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"alert","signature_id":"a34c4272b290720aafd3b29a745fe60b6976f6c8d6360946f5a75edf1096c9ec","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 24 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T01:01:07Z\n Ended: 2026-09-21T01:19:09Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T01:01:07Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-21T01:03:05Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:17:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-21T01:05:07Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-21T01:05:07Z\n Ended: 2026-09-21T01:17:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-21T01:05:07Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-21T01:06:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-21T01:06:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":17,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":17,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7036,"pr_state":"open","pr_sha":"ea3a7b1eaf473ee4001099d59d4391c719fb5e27","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T23:13:50Z","started_at":"2026-09-20T23:13:50Z","completed_at":"2026-09-21T02:00:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T02:12:59.946011228Z","timing_metadata_checked_at":"2026-09-21T02:12:59.946011228Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T02:12:59.950752412Z","region_metadata_checked_at":"2026-09-21T02:12:59.950752412Z"},"failed_at":["alert","e2e"],"failed_test_count":10,"occurrences":[{"row_id":"18f206f957693e37805ca819b3ae56f55cbe0dd9fd9987b685785a052f4bc7e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"b10599bdf4bc1308f14fce45589e487e1d0d085cf0ef4c4bb64b2cf8c30f79c9","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:18:44Z\n Ended: 2026-09-21T00:50:46Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4251392-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4251392-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"307d540e5306bdda70080fb027acad3479c988d7a01d001d417c1ed268281bfa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"e2e","signature_id":"ae6ec300b2e1b5ec89e4505b36ba0351611d82380af69c6bc2aa2a3220759883","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001838090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s2nb9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-2wc8bnqpjvsw\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"name\": \"3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:41.012625795Z\",\n \"endTime\": \"2026-09-21T01:06:41.448864721Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2hu8mbpcvgabffnu4e7eeji5v72ts still exists (deletion dispatched at 2026-09-21T00:42:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001838090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s2nb9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-2wc8bnqpjvsw\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"name\": \"3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:41.012625795Z\",\n \"endTime\": \"2026-09-21T01:06:41.448864721Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2hu8mbpcvgabffnu4e7eeji5v72ts still exists (deletion dispatched at 2026-09-21T00:42:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3c618356549149cc506e7521d50594e732435aa60cd4196dbda5c35f2f89b2e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"258bf6b8e2e6a15e2ce38412cd2942fddf00da1a19bafe63316ac35a44ba6d7c","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:30:39Z\n Ended: 2026-09-21T00:36:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-5q82q\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-5q82q has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"3e31181738ed75ee589a6f8117ffad835df22228825cd21f4eafce5987fecd67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"e2e","signature_id":"297f5e4b1d1f4952837a1cae50b737b42a9bc431965f0ebd876867d9c038a0d0","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100fa80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-xdxwcdw5rptk\" to finish creating: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"name\": \"f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.938506094Z\",\n \"endTime\": \"2026-09-21T00:46:21.511326736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100fa80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-xdxwcdw5rptk\" to finish creating: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"name\": \"f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.938506094Z\",\n \"endTime\": \"2026-09-21T00:46:21.511326736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"63f71d1d7a41f3919262f729189d1af4f7d04a490ad23859344b6ac9b56c4695","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"39e21beab76cee821e0ed80a6601d86d04d54351b87f382522d67356ee5fa584","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-21T01:28:40Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4251392-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5hkmr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-5hkmr (namespace arobit) on cluster ci01-j4251392-svc has averaged 126.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"aa4fdcacd8a4f86701323d44d124a044f1569a10b0ef847537ac018f296f852a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"e2e","signature_id":"eac5055c826c77b923258da473791d4fdf5723c4be718ddb37bd909244aa07ad","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e816b0\u003e: \n GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01726526-88f4-4112-af3c-fe53fa8a4dfc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01726526-88f4-4112-af3c-fe53fa8a4dfc\",\n \"name\": \"01726526-88f4-4112-af3c-fe53fa8a4dfc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:15:41.81594577Z\",\n \"endTime\": \"2026-09-21T00:44:49.261787828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e816b0\u003e: \n GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01726526-88f4-4112-af3c-fe53fa8a4dfc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01726526-88f4-4112-af3c-fe53fa8a4dfc\",\n \"name\": \"01726526-88f4-4112-af3c-fe53fa8a4dfc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:15:41.81594577Z\",\n \"endTime\": \"2026-09-21T00:44:49.261787828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c0c81a7db0f8182ea382538c1371269fb7760f86f36c3b561ea2b28af2067a9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"a2c1fbf6e64cddf7a46eaafcc044b53f14f98a5940a39f4dbc58486310e950ce","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:49:52Z\n Ended: 2026-09-21T01:04:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"c1c9b0d6c1b5161e32c02d67d64772380d9e043b8f2c62a14cb2358483dd6a88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"386fa5f6fc68894d00954640db0f0852279a2e3a87df5f417fecec60fd94a6bb","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:35:55Z\n Ended: 2026-09-21T01:02:51Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:35:55Z\n Ended: 2026-09-21T01:06:53Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"c3ed493ac8c0bca47194ee7713e183de4467675615c56e7e290ac9691ecba846","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"14896a8781a6fade79c483f0daff72f9c39c1157a6161055b98515a6ef455b60","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:38:51Z\n Ended: 2026-09-21T01:06:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4d11d27a-526b-40d4-812b-9a43bae7deb5\"\n Description: Pod ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"d80da60103efca7bf98933e0aae4885b139db7e5ec96b7965e39bdb976d61de1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"e2e","signature_id":"48429fa1e1b278e83c9c1235be2155e15cb94e91feb6a91189f07417aac2381b","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00139c1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kctnzvxjhvcd\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"name\": \"3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:02:07.101805352Z\",\n \"endTime\": \"2026-09-21T01:26:13.044715763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2i1qqe304gugno1vb23hj27er6drc still exists (deletion dispatched at 2026-09-21T01:02:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00139c1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kctnzvxjhvcd\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"name\": \"3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:02:07.101805352Z\",\n \"endTime\": \"2026-09-21T01:26:13.044715763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2i1qqe304gugno1vb23hj27er6drc still exists (deletion dispatched at 2026-09-21T01:02:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101810235566985216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7035,"pr_state":"closed","pr_sha":"18faff8c97058d15283061028cf5dd67da4bc619","final_merged_sha":"18faff8c97058d15283061028cf5dd67da4bc619","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T23:06:24Z","started_at":"2026-09-20T23:06:24Z","completed_at":"2026-09-20T23:17:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T23:35:24.284318914Z","timing_metadata_checked_at":"2026-09-20T23:35:24.284318914Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-20T23:35:24.288045202Z","region_metadata_checked_at":"2026-09-21T00:39:21.186712485Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a924fd5b7bad3d10014d8a7c100fb24a3231a372881f4791a694a399180d3779","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101810235566985216","occurred_at":"2026-09-20T23:06:24Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T23:02:29Z","started_at":"2026-09-20T23:02:29Z","completed_at":"2026-09-21T01:27:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T01:41:37.432073396Z","timing_metadata_checked_at":"2026-09-21T01:41:37.432073396Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T01:41:37.437354879Z","region_metadata_checked_at":"2026-09-21T01:41:37.437354879Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"61a2299daea1fbe35ab0d03e99cf5d7fd82287b115bedf93671f4fde23ca9211","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","failed_at":"e2e","signature_id":"e4aee9aff864d2f43ab51682459e47fa9b2962afca634816bc039aba490af5e3","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00174e0c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k6sk5ztcb6st\" to finish deleting: GET https://rp.j7913600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"name\": \"2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:34.467872497Z\",\n \"endTime\": \"2026-09-21T01:06:34.900855786Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2gfj01vbki6vfv5k1lnt4j81kjqnb still exists (deletion dispatched at 2026-09-21T00:42:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00174e0c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k6sk5ztcb6st\" to finish deleting: GET https://rp.j7913600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"name\": \"2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:34.467872497Z\",\n \"endTime\": \"2026-09-21T01:06:34.900855786Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2gfj01vbki6vfv5k1lnt4j81kjqnb still exists (deletion dispatched at 2026-09-21T00:42:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9d31c92412593ce168593ff6bfd5c137d2f457185edca3524390039b0901cc0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","failed_at":"alert","signature_id":"7d106244608ace8de6f9cae03dbed9bf2952c22134ad7882a0e61735843e27ac","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:24:26Z\n Ended: 2026-09-21T00:30:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7913600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2ggt01co6d2ehgcrfpefiv7co64vf-v20260901\", pod=\"router-fbcbc989-n4pcs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2ggt01co6d2ehgcrfpefiv7co64vf-v20260901/router-fbcbc989-n4pcs has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:24:27Z\n Ended: 2026-09-21T00:30:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7913600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2ggpos1dbpnrhtfim3df37oqp4d6v-q4q2f0t2i0p0l7c\", pod=\"router-cb77f855-wpm9t\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2ggpos1dbpnrhtfim3df37oqp4d6v-q4q2f0t2i0p0l7c/router-cb77f855-wpm9t has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c7925f2d46db8ff9bd3aa9315850ebe844918a61877de618c7ea3a8b88e94dba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","failed_at":"alert","signature_id":"f65d23db3fce211d34d0769e73e3f8f7acba651c0170fe2e1546f79d4a71b579","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:42:36Z\n Ended: 2026-09-21T00:48:35Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7913600-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7913600-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101793807640039424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7035,"pr_state":"closed","pr_sha":"18faff8c97058d15283061028cf5dd67da4bc619","final_merged_sha":"18faff8c97058d15283061028cf5dd67da4bc619","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T22:01:03Z","started_at":"2026-09-20T22:01:03Z","completed_at":"2026-09-20T22:10:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:20:34.218887245Z","timing_metadata_checked_at":"2026-09-20T22:20:34.218887245Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-20T22:20:34.221789635Z","region_metadata_checked_at":"2026-09-20T23:24:25.230688459Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5f8a171339e4fa4b01cb33f8a405fb8e94d088d0b6faaf89b8949f02006768a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101793807640039424","occurred_at":"2026-09-20T22:01:03Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101793797569515520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7032,"pr_state":"closed","pr_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","final_merged_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T22:00:55Z","started_at":"2026-09-20T22:00:55Z","completed_at":"2026-09-20T22:10:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:20:34.035755665Z","timing_metadata_checked_at":"2026-09-20T22:20:34.035755665Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-20T22:20:34.039113153Z","region_metadata_checked_at":"2026-09-20T23:24:25.10530806Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"07eb0d0cf2b8480cb9a23375c12127b091d5b39fbbe3335b3909f8463082131e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101793797569515520","occurred_at":"2026-09-20T22:00:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101767145149960192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7033,"pr_state":"closed","pr_sha":"c448a0db20d77ea047c00534044b12cc2b2ac565","final_merged_sha":"ef821496d20065a3db001282a41d83827d994051","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T20:15:04Z","started_at":"2026-09-20T20:15:04Z","completed_at":"2026-09-20T22:20:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:31:19.992970333Z","timing_metadata_checked_at":"2026-09-20T22:31:19.992970333Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T22:31:19.995885724Z","region_metadata_checked_at":"2026-09-20T22:31:19.995885724Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4fb4380e15dfd4cd63886dfa7fbfc67d4248bb08efd96f7c38b97cd4da166065","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101767145149960192","occurred_at":"2026-09-20T20:15:04Z","failed_at":"e2e","signature_id":"55fec78f783ce8cf181e98d25b36b712ff6eab60ce5978dd2cbcd0aee43a639c","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001dac330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lfwqsfxv46jf\" to finish deleting: GET https://rp.j9960192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11bbccb6-f070-4edf-a07d-e5d2b5dbade5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11bbccb6-f070-4edf-a07d-e5d2b5dbade5\",\n \"name\": \"11bbccb6-f070-4edf-a07d-e5d2b5dbade5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:32:16.596553994Z\",\n \"endTime\": \"2026-09-20T21:56:23.901016482Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvlb6tjn805avjlrq54um1k9ghb2e1 still exists (deletion dispatched at 2026-09-20T21:32:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001dac330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lfwqsfxv46jf\" to finish deleting: GET https://rp.j9960192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11bbccb6-f070-4edf-a07d-e5d2b5dbade5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11bbccb6-f070-4edf-a07d-e5d2b5dbade5\",\n \"name\": \"11bbccb6-f070-4edf-a07d-e5d2b5dbade5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:32:16.596553994Z\",\n \"endTime\": \"2026-09-20T21:56:23.901016482Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvlb6tjn805avjlrq54um1k9ghb2e1 still exists (deletion dispatched at 2026-09-20T21:32:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f4197d0de3962b76d7915e149600df79d9d5574af1a8b654b80991c16c6a2df3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101767145149960192","occurred_at":"2026-09-20T20:15:04Z","failed_at":"e2e","signature_id":"753848c3bc300eb5da8ad64aa08adc3a8b8dafee7fb505106b6f5cbf501ef653","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011c8078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-c465zc8xkqx7\" to finish deleting: GET https://rp.j9960192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ec0221a2-95f2-4368-b530-97209abb26ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ec0221a2-95f2-4368-b530-97209abb26ab\",\n \"name\": \"ec0221a2-95f2-4368-b530-97209abb26ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:41:10.549898621Z\",\n \"endTime\": \"2026-09-20T22:05:13.934972715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvm850d5khb2asjdadkcgoks2iqobg still exists (deletion dispatched at 2026-09-20T21:41:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011c8078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-c465zc8xkqx7\" to finish deleting: GET https://rp.j9960192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ec0221a2-95f2-4368-b530-97209abb26ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ec0221a2-95f2-4368-b530-97209abb26ab\",\n \"name\": \"ec0221a2-95f2-4368-b530-97209abb26ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:41:10.549898621Z\",\n \"endTime\": \"2026-09-20T22:05:13.934972715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvm850d5khb2asjdadkcgoks2iqobg still exists (deletion dispatched at 2026-09-20T21:41:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T19:59:59Z","started_at":"2026-09-20T19:59:59Z","completed_at":"2026-09-20T22:01:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:10:34.004824246Z","timing_metadata_checked_at":"2026-09-20T22:10:34.004824246Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T22:10:34.009069832Z","region_metadata_checked_at":"2026-09-20T22:10:34.009069832Z"},"failed_at":["alert","e2e"],"failed_test_count":7,"occurrences":[{"row_id":"2ab909dd4e9b4f90b480bb53ab5308e59e6dbd540d30ccbb5717b9b15410e23c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"571f2e2e0cd4941242495f0f83848c90e92c491345357a2fc6f06f0266aa0a09","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef5080\u003e: \n DNS for route host agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef5080\u003e: \n DNS for route host agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"40c5c3ae1a4dd2e4450450ab1130526b102a6263cd4db10ae8beb349e5923494","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"ae135f0135bd68e7f6075a32b2c72f3bdce2c27a9f86f02269bf9ec0d9c29b8b","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002510348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-w9lxmqgrgjxc\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\",\n \"name\": \"d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:00:38.791668601Z\",\n \"endTime\": \"2026-09-20T21:24:44.133089654Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvaqju2imrg5degq03jceicg07q1sk still exists (deletion dispatched at 2026-09-20T21:00:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002510348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-w9lxmqgrgjxc\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\",\n \"name\": \"d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:00:38.791668601Z\",\n \"endTime\": \"2026-09-20T21:24:44.133089654Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvaqju2imrg5degq03jceicg07q1sk still exists (deletion dispatched at 2026-09-20T21:00:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"82b02cb558491a0dfcb05a6312dc71ada50ccd9673e699b6e60a63f70a348374","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"1de82b4d07b3d460f0773709ee7a723eb163f891926727a6696a1f16b19d90b5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015b00f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-x24g9c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-zwzt7v9n4qld\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d039276d-bda2-483a-a789-67d1043bffad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d039276d-bda2-483a-a789-67d1043bffad\",\n \"name\": \"d039276d-bda2-483a-a789-67d1043bffad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:01:45.790091965Z\",\n \"endTime\": \"2026-09-20T21:25:54.10177225Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvajhucng7kcjeavelijr1m7gh4dvh still exists (deletion dispatched at 2026-09-20T21:01:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015b00f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-x24g9c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-zwzt7v9n4qld\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d039276d-bda2-483a-a789-67d1043bffad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d039276d-bda2-483a-a789-67d1043bffad\",\n \"name\": \"d039276d-bda2-483a-a789-67d1043bffad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:01:45.790091965Z\",\n \"endTime\": \"2026-09-20T21:25:54.10177225Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvajhucng7kcjeavelijr1m7gh4dvh still exists (deletion dispatched at 2026-09-20T21:01:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"84fe011b53eefdd61e326845fe8c39c49cb1074ea1ddc8276bc312f378bff940","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"6b62ee5741da84d36547bbd46f0d60714cb4fa8627bb19d966f92a3eba03e28a","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0018b60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-gmw544q97wff\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df6c4988-d57f-4cfa-89da-42c4c8598408\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df6c4988-d57f-4cfa-89da-42c4c8598408\",\n \"name\": \"df6c4988-d57f-4cfa-89da-42c4c8598408\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:20:32.391377317Z\",\n \"endTime\": \"2026-09-20T21:44:34.405759905Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvarbr3muag22o8gt4v9kc9nb03f6d still exists (deletion dispatched at 2026-09-20T21:20:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0018b60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-gmw544q97wff\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df6c4988-d57f-4cfa-89da-42c4c8598408\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df6c4988-d57f-4cfa-89da-42c4c8598408\",\n \"name\": \"df6c4988-d57f-4cfa-89da-42c4c8598408\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:20:32.391377317Z\",\n \"endTime\": \"2026-09-20T21:44:34.405759905Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvarbr3muag22o8gt4v9kc9nb03f6d still exists (deletion dispatched at 2026-09-20T21:20:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9582bd4cd943f84a97b8a37fbf59f6d65e52be2f5249fc051e1787508a7ad991","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"35c9e2930b67189ab544a31c2246d603785a353beb1727154337c02311b5ba68","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001846078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-8xl9z7\" in resourcegroup=\"rg-zstream-upgrade-4-21-8m66bndkns9t\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3409ee9c-20cd-4a04-bbea-059ca801b95b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3409ee9c-20cd-4a04-bbea-059ca801b95b\",\n \"name\": \"3409ee9c-20cd-4a04-bbea-059ca801b95b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:01:55.419339651Z\",\n \"endTime\": \"2026-09-20T21:26:04.130585156Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvahv650fl61a4dvgvtu796fkako2j still exists (deletion dispatched at 2026-09-20T21:02:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001846078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-8xl9z7\" in resourcegroup=\"rg-zstream-upgrade-4-21-8m66bndkns9t\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3409ee9c-20cd-4a04-bbea-059ca801b95b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3409ee9c-20cd-4a04-bbea-059ca801b95b\",\n \"name\": \"3409ee9c-20cd-4a04-bbea-059ca801b95b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:01:55.419339651Z\",\n \"endTime\": \"2026-09-20T21:26:04.130585156Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvahv650fl61a4dvgvtu796fkako2j still exists (deletion dispatched at 2026-09-20T21:02:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d2d359344b56f2e35dc32c77172eebf32fe5bc0e3f1e55a7e3faf27395528500","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"alert","signature_id":"8f4619349028d8271e222f8d6571b2f090f7d7878ecb45ed7f028937641f406f","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T21:05:56Z\n Ended: 2026-09-20T21:11:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8645120-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8645120-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"e563a12983d7f7f9811f114d67585d0c17e21f3dcdb0a71f8e6024c18aded0e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"6ecc7b9cabb4ec501df9af223046a31e6f46790bc97afea6159cc8363fead315","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001418108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-zdswd5\" in resourcegroup=\"rg-zstream-upgrade-4-20-xsntgddnxfll\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ba72035-9acd-4b81-9004-66ff1293e676\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ba72035-9acd-4b81-9004-66ff1293e676\",\n \"name\": \"3ba72035-9acd-4b81-9004-66ff1293e676\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:05:10.531590012Z\",\n \"endTime\": \"2026-09-20T21:29:14.121060804Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvamm4ck5qen7rpohu5h53k3sludes still exists (deletion dispatched at 2026-09-20T21:05:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001418108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-zdswd5\" in resourcegroup=\"rg-zstream-upgrade-4-20-xsntgddnxfll\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ba72035-9acd-4b81-9004-66ff1293e676\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ba72035-9acd-4b81-9004-66ff1293e676\",\n \"name\": \"3ba72035-9acd-4b81-9004-66ff1293e676\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:05:10.531590012Z\",\n \"endTime\": \"2026-09-20T21:29:14.121060804Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvamm4ck5qen7rpohu5h53k3sludes still exists (deletion dispatched at 2026-09-20T21:05:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101762182411718656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7035,"pr_state":"closed","pr_sha":"18faff8c97058d15283061028cf5dd67da4bc619","final_merged_sha":"18faff8c97058d15283061028cf5dd67da4bc619","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T19:55:27Z","started_at":"2026-09-20T19:55:27Z","completed_at":"2026-09-20T21:45:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:00:09.160560343Z","timing_metadata_checked_at":"2026-09-20T22:00:09.160560343Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T22:00:09.164119432Z","region_metadata_checked_at":"2026-09-20T22:00:09.164119432Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6bc834e630c0a71f4bddc3195d298b758f1630f57278854ea502bfbb588474dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101762182411718656","occurred_at":"2026-09-20T19:55:27Z","failed_at":"alert","signature_id":"993e0edbc2802d27b58b7315a4badc5470fe6efc5c6616983981032abe8b49db","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T21:15:25Z\n Ended: 2026-09-20T21:27:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j1718656-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j1718656-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7030,"pr_state":"closed","pr_sha":"4dc2c443d9d2f9fa2a598585253f8231c7ddf46c","final_merged_sha":"50f9710d6eb949ad5cf7c325f7c4d6cce9083ddf","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T19:26:34Z","started_at":"2026-09-20T19:26:34Z","completed_at":"2026-09-20T22:02:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:10:34.12856235Z","timing_metadata_checked_at":"2026-09-20T22:10:34.12856235Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T22:10:34.13160584Z","region_metadata_checked_at":"2026-09-20T22:10:34.13160584Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"04ffcaf7762f47ecf73e0578e98552f5ce37fe12b44c37983f7dca3c8ba7c9f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","failed_at":"alert","signature_id":"e7c54520064dc8897e885bf4790b5cdcc177d7e87c6cdbf61692ea5647d049d2","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T21:06:19Z\n Ended: 2026-09-20T21:13:22Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7010688-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7010688-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"42a23fe7038468a62121df425e6e6a85251f017045c5cfb333451071e07732f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","failed_at":"e2e","signature_id":"94c88bfe25b6cc5073858901db97efc9bfdacd7a00fdfffa45d0abf7a6f10cff","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00122c0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-wkvn4m8rs9f7\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c840051-bc7e-4300-9c8b-cdc5ba02f524\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c840051-bc7e-4300-9c8b-cdc5ba02f524\",\n \"name\": \"5c840051-bc7e-4300-9c8b-cdc5ba02f524\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:21:26.975187286Z\",\n \"endTime\": \"2026-09-20T21:45:29.127892727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvbff7c84adptpfq9s8m0tpv7v5ddl still exists (deletion dispatched at 2026-09-20T21:21:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00122c0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-wkvn4m8rs9f7\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c840051-bc7e-4300-9c8b-cdc5ba02f524\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c840051-bc7e-4300-9c8b-cdc5ba02f524\",\n \"name\": \"5c840051-bc7e-4300-9c8b-cdc5ba02f524\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:21:26.975187286Z\",\n \"endTime\": \"2026-09-20T21:45:29.127892727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvbff7c84adptpfq9s8m0tpv7v5ddl still exists (deletion dispatched at 2026-09-20T21:21:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"51401fbe99f3cf3dd11c2ce463f61972b770f7fcc35f4cd2de936f121e2072bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","failed_at":"e2e","signature_id":"ba4ec05dffbf7aeb4b673f4a633bc6bb64ed25fa0b09ea36f4467a08976b4236","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a7c258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-8wrhrq\" in resourcegroup=\"rg-zstream-upgrade-4-22-sbnm9pqkcwf6\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\",\n \"name\": \"2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:02:20.578949773Z\",\n \"endTime\": \"2026-09-20T21:26:20.682855256Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvahe0tlai0bmukqovr96jjrqs3mbm still exists (deletion dispatched at 2026-09-20T21:02:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a7c258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-8wrhrq\" in resourcegroup=\"rg-zstream-upgrade-4-22-sbnm9pqkcwf6\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\",\n \"name\": \"2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:02:20.578949773Z\",\n \"endTime\": \"2026-09-20T21:26:20.682855256Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvahe0tlai0bmukqovr96jjrqs3mbm still exists (deletion dispatched at 2026-09-20T21:02:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"88842db398506118e17a77b985575c009ec44fea0a9fcda307afe67f4b3c8c2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","failed_at":"e2e","signature_id":"5e8455d974423d74da7a2764a6de9171d767628d73b6837a9e7152a9dc31c031","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016aa0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-76bn77\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-8cjn4vhljp95\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddd0e4ac-a249-441c-b385-574e70d538ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddd0e4ac-a249-441c-b385-574e70d538ac\",\n \"name\": \"ddd0e4ac-a249-441c-b385-574e70d538ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:02:37.113406649Z\",\n \"endTime\": \"2026-09-20T21:26:40.645444433Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvagov3s5e6i263h39l67869g6anai still exists (deletion dispatched at 2026-09-20T21:02:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016aa0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-76bn77\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-8cjn4vhljp95\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddd0e4ac-a249-441c-b385-574e70d538ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddd0e4ac-a249-441c-b385-574e70d538ac\",\n \"name\": \"ddd0e4ac-a249-441c-b385-574e70d538ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:02:37.113406649Z\",\n \"endTime\": \"2026-09-20T21:26:40.645444433Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvagov3s5e6i263h39l67869g6anai still exists (deletion dispatched at 2026-09-20T21:02:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d48ecf48fc0366e4f72b9887aa18aac2c837aeebe1466f6afaba7d9289ee5693","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","failed_at":"e2e","signature_id":"fec305f359038a8b537a558ba87b3d6d7b2c427d51eb493f15dad5445be91e77","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c6e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-hxgb2rxvk82d\" to finish creating: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"name\": \"2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:48:17.088169395Z\",\n \"endTime\": \"2026-09-20T21:07:20.582868448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c6e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-hxgb2rxvk82d\" to finish creating: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"name\": \"2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:48:17.088169395Z\",\n \"endTime\": \"2026-09-20T21:07:20.582868448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7032,"pr_state":"closed","pr_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","final_merged_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T19:01:41Z","started_at":"2026-09-20T19:01:41Z","completed_at":"2026-09-20T21:18:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T21:28:28.34717848Z","timing_metadata_checked_at":"2026-09-20T21:28:28.34717848Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T21:28:28.350753869Z","region_metadata_checked_at":"2026-09-20T21:28:28.350753869Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"1b8ffee88db2d9644f214d173b10667549564aa79636c133c0bb85385e5386e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"204e1750cfbcdccc7a1aa87cf8b29089bd7645257cbc1509ff43f3d1c64bf5f6","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:518]: Timed out after 900.001s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:505 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: ...\nto be \u003e=\n \u003cint\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:518]: Timed out after 900.001s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:505 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: ...\nto be \u003e=\n \u003cint\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have at least 2 nodes (autoscaling min), found 0","search_query":"node pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0"}},{"row_id":"251bab08c74f242248bf8b408cbe65e12cc630f8f6eb4fa5426fd6c014c4cba4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"08debcb95fe76787cc5e7470c732a78966ba147069573bb898c775a88f1cff3e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000219dc0\u003e: \n failed to create NodePool np-ggj24g: failed waiting for nodepool=\"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\" in resourcegroup=\"rg-candidate-4-20-686sncjvb2sx\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"name\": \"60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:06:27.477664357Z\",\n \"endTime\": \"2026-09-20T20:25:30.377289875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000219dc0\u003e: \n failed to create NodePool np-ggj24g: failed waiting for nodepool=\"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\" in resourcegroup=\"rg-candidate-4-20-686sncjvb2sx\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"name\": \"60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:06:27.477664357Z\",\n \"endTime\": \"2026-09-20T20:25:30.377289875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b3810171f30121cd8713e3d56756d3faec14cf7125baae340883643c8ad6a1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"48929c5f392dda19c39d764a9d4117d26ea3b70f835a97462a0249c92cc9fdf4","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"422ddc32f327db7a61c881b4411ef7767ec79e98b46476bd6a0cffdccf708740","failure_pattern":"expected all nodes to have label 'key1=value1'","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'"}},{"row_id":"55d7f598d1a41824505461f5e95aaf17529ca46721083fb901c92de616f0db9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade"}},{"row_id":"594547218134bd8028a5e91412c7f97d36bb85dd140aad97e46b8b6d20fd6422","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"f61d48f7e4dfc0d5ed670d4815de996c6e11f3ee6013821737755d17d338fd53","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010007a0\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010007a0\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"55fc315450b35ec48de3526d899619e34f6aae1d138260064fe2d53d5403f86e","failure_pattern":"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet \u003cnamespace\u003e/global-pull-secret-syncer has no desired pods scheduled","search_query":"occurred"}},{"row_id":"84af0e38f504268297a3db4476d8fd473b54efa2fa564c726e527b345e479498","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"e54b973f5d0dfed8acc1e8e3583260026a6a968d1ae6ee942caf592be98feaeb","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe67a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-9p6vvl\" in resourcegroup=\"rg-np-noedge-9p6vvl-v6jt8npv8wnh\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3713b0f0-249d-4350-9997-27de0131dfcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3713b0f0-249d-4350-9997-27de0131dfcd\",\n \"name\": \"3713b0f0-249d-4350-9997-27de0131dfcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:02:41.432631359Z\",\n \"endTime\": \"2026-09-20T20:21:50.467881337Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe67a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-9p6vvl\" in resourcegroup=\"rg-np-noedge-9p6vvl-v6jt8npv8wnh\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3713b0f0-249d-4350-9997-27de0131dfcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3713b0f0-249d-4350-9997-27de0131dfcd\",\n \"name\": \"3713b0f0-249d-4350-9997-27de0131dfcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:02:41.432631359Z\",\n \"endTime\": \"2026-09-20T20:21:50.467881337Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9eab0a1eddffa2e0aecd4cf75a6ac1524838cc900ecd0dce5d321c8fb6c6f950","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"db98f57638dcde7ae6924500ef05678fc197bea0143f095eee401504ac389cb8","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00199a0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-mr4hsq\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-tl7tb6ttppj7\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9eafb140-dd1b-4c27-a86b-1a1997869e11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9eafb140-dd1b-4c27-a86b-1a1997869e11\",\n \"name\": \"9eafb140-dd1b-4c27-a86b-1a1997869e11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:20:56.231911113Z\",\n \"endTime\": \"2026-09-20T20:45:00.42480766Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suungo1m007b7a8qlj4ggi47d944qra still exists (deletion dispatched at 2026-09-20T20:20:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00199a0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-mr4hsq\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-tl7tb6ttppj7\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9eafb140-dd1b-4c27-a86b-1a1997869e11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9eafb140-dd1b-4c27-a86b-1a1997869e11\",\n \"name\": \"9eafb140-dd1b-4c27-a86b-1a1997869e11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:20:56.231911113Z\",\n \"endTime\": \"2026-09-20T20:45:00.42480766Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suungo1m007b7a8qlj4ggi47d944qra still exists (deletion dispatched at 2026-09-20T20:20:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aa4dcd6f2765b96ab3c2fd68d918016ea1ed6c40dac4a365a21fd17d85456c4b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"81d3fa9d4a49bf594db48e69cfb405302d6d567f873fa4a49e7a724e1f6a4553","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a82108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-vctfds\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-kw9j54cmg4pc\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4e37d117-e332-4860-a681-21465cb9fec9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4e37d117-e332-4860-a681-21465cb9fec9\",\n \"name\": \"4e37d117-e332-4860-a681-21465cb9fec9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:23:12.755096811Z\",\n \"endTime\": \"2026-09-20T20:47:20.418777117Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suunh30duok5htrpqis3a229ruh2gu8 still exists (deletion dispatched at 2026-09-20T20:23:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a82108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-vctfds\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-kw9j54cmg4pc\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4e37d117-e332-4860-a681-21465cb9fec9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4e37d117-e332-4860-a681-21465cb9fec9\",\n \"name\": \"4e37d117-e332-4860-a681-21465cb9fec9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:23:12.755096811Z\",\n \"endTime\": \"2026-09-20T20:47:20.418777117Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suunh30duok5htrpqis3a229ruh2gu8 still exists (deletion dispatched at 2026-09-20T20:23:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c20ff8783765f263971647de32041ce98788190a9255de24759765697c8d22be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"531647f22030c4eb57aa8d0ce0f1192ad57dde45a170d15dbfbc6d84af797341","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dfc100\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-wbdbqt\" in resourcegroup=\"rg-np-version-upgrade-wbdbqt-c4kwcbz5jpzk\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b9c3e353-8a9e-429f-b72a-d1506b35ff09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b9c3e353-8a9e-429f-b72a-d1506b35ff09\",\n \"name\": \"b9c3e353-8a9e-429f-b72a-d1506b35ff09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:03:32.070997575Z\",\n \"endTime\": \"2026-09-20T20:22:40.415779239Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dfc100\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-wbdbqt\" in resourcegroup=\"rg-np-version-upgrade-wbdbqt-c4kwcbz5jpzk\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b9c3e353-8a9e-429f-b72a-d1506b35ff09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b9c3e353-8a9e-429f-b72a-d1506b35ff09\",\n \"name\": \"b9c3e353-8a9e-429f-b72a-d1506b35ff09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:03:32.070997575Z\",\n \"endTime\": \"2026-09-20T20:22:40.415779239Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cb9dc326990e2c6c26ac44245386ef91018e5f17d25ea90d03967f27567362f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"de84625bbe06b7fedfd12cf6c287f90f560af49bcb2f31d73c8b14a307cfe92d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:474]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:474]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"e22addc8ad702d24529905c2d8ecae8bc9e8ecb5f14bedb2fa4043c989ebd7b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"6430a6d078ca6a6f372c5a118c728de07a27f33f7e2dd5e2fdf9a7c60c12d6f7","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00126e0f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-4rwkxxhrqmpn\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/67887784-f8ef-49d8-9a3a-e8734d5e320e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/67887784-f8ef-49d8-9a3a-e8734d5e320e\",\n \"name\": \"67887784-f8ef-49d8-9a3a-e8734d5e320e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:38:02.27895881Z\",\n \"endTime\": \"2026-09-20T21:02:10.843079298Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suunh6uiggfn6q1bvsp3td8abp45hh0 still exists (deletion dispatched at 2026-09-20T20:38:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00126e0f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-4rwkxxhrqmpn\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/67887784-f8ef-49d8-9a3a-e8734d5e320e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/67887784-f8ef-49d8-9a3a-e8734d5e320e\",\n \"name\": \"67887784-f8ef-49d8-9a3a-e8734d5e320e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:38:02.27895881Z\",\n \"endTime\": \"2026-09-20T21:02:10.843079298Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suunh6uiggfn6q1bvsp3td8abp45hh0 still exists (deletion dispatched at 2026-09-20T20:38:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["422ddc32f327db7a61c881b4411ef7767ec79e98b46476bd6a0cffdccf708740","55fc315450b35ec48de3526d899619e34f6aae1d138260064fe2d53d5403f86e","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T17:10:13Z","started_at":"2026-09-20T17:10:13Z","completed_at":"2026-09-20T19:14:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T19:23:26.956472331Z","timing_metadata_checked_at":"2026-09-20T19:23:26.956472331Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T19:23:26.959900721Z","region_metadata_checked_at":"2026-09-20T19:23:26.959900721Z"},"failed_at":["alert","e2e"],"failed_test_count":15,"occurrences":[{"row_id":"0122f5826abd42c43d62e9f73009fb5437a4c3ba79541255dd70929cb08074bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"alert","signature_id":"5e756453c54bd40ab2d5dd229f82f628f7919ef41eabbc02f3d959659f4f7234","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 20 time(s) (10 unknown, 10 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-20T18:00:47Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-20T18:01:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"2185a07f705f9f3fadeea684cf0d4241fd07abdfbdd7f67261fa1ac69542e62f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"0122ecc6da0d7f29528efc4c03aacfe2a95abd22053ee427b3ba034f89f24578","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012a4340\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-lzrwjdpxp8w2\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b173996-7453-4b97-b67f-ef797fded833\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b173996-7453-4b97-b67f-ef797fded833\",\n \"name\": \"6b173996-7453-4b97-b67f-ef797fded833\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:01.158204247Z\",\n \"endTime\": \"2026-09-20T18:19:10.424114875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012a4340\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-lzrwjdpxp8w2\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b173996-7453-4b97-b67f-ef797fded833\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b173996-7453-4b97-b67f-ef797fded833\",\n \"name\": \"6b173996-7453-4b97-b67f-ef797fded833\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:01.158204247Z\",\n \"endTime\": \"2026-09-20T18:19:10.424114875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3175ba2f730a74105a2c93536a32d55a63f01b5e74618ec26e64c9d1604344dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"bf5a3ba4ae1888ec5bbf4c1ea6088363a29b33944bb08384b179d0e7edc222cb","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d64220\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-d5dfblc9jxtp\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa297b6b-fe8e-4087-b659-ba08e6f85c73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa297b6b-fe8e-4087-b659-ba08e6f85c73\",\n \"name\": \"aa297b6b-fe8e-4087-b659-ba08e6f85c73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:51:22.937689743Z\",\n \"endTime\": \"2026-09-20T18:20:30.411667073Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d64220\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-d5dfblc9jxtp\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa297b6b-fe8e-4087-b659-ba08e6f85c73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa297b6b-fe8e-4087-b659-ba08e6f85c73\",\n \"name\": \"aa297b6b-fe8e-4087-b659-ba08e6f85c73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:51:22.937689743Z\",\n \"endTime\": \"2026-09-20T18:20:30.411667073Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"47f9005bee847c29df7beaebcb5920253cf086a2102dc9dbbf3b1e84e0c16db1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"e6538054ab0559cddb2c0ed63235c48a4d5cd57bef25e042c4c66fb7e93ba92e","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c0f60\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2fppphk96nck\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"name\": \"648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:38.502314717Z\",\n \"endTime\": \"2026-09-20T18:19:40.439163186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c0f60\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2fppphk96nck\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"name\": \"648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:38.502314717Z\",\n \"endTime\": \"2026-09-20T18:19:40.439163186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"44fc5c170ee5969da686dbaeb61a39c91b4e9f01cf7a4324b16151d7f8a7811a","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4c22a1b01844201cfcfdc984f4b575d79c5ea412e8f882ddf242bacf6042887a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"a2be2cf8041303505466850a3cb9c21d1f99bbb048308b395ef9642bc9197ca2","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6c600\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-gc4d4nn6mhwj\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7556a704-973b-41e7-a11a-80eff56506d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7556a704-973b-41e7-a11a-80eff56506d4\",\n \"name\": \"7556a704-973b-41e7-a11a-80eff56506d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:01.159372023Z\",\n \"endTime\": \"2026-09-20T18:19:10.475238514Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6c600\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-gc4d4nn6mhwj\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7556a704-973b-41e7-a11a-80eff56506d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7556a704-973b-41e7-a11a-80eff56506d4\",\n \"name\": \"7556a704-973b-41e7-a11a-80eff56506d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:01.159372023Z\",\n \"endTime\": \"2026-09-20T18:19:10.475238514Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4d9776443d827be3c6e37637e5df0c2abec7249241c02712046df48bf05ae55e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"alert","signature_id":"b4c5714662b6eebf9aeab7aa2383a89d5f76557fa6ae1c0f2264a9d31c883f89","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-20T18:55:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3380352-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qgqnm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-qgqnm (namespace arobit) on cluster ci01-j3380352-svc has averaged 126.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"6548e68cfb1ff1fb19544ccfcc21aee5769c5c07ab8a8e3dc07e7de01b8dff0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"66049a9e3aec5154c649a5163e0d3ee1db514ce22e0071326628d432e4a129e1","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00138a198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-x79rddkrff4p\" to finish deleting: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5321d526-4d45-49af-b781-508e5635a670\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5321d526-4d45-49af-b781-508e5635a670\",\n \"name\": \"5321d526-4d45-49af-b781-508e5635a670\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T18:34:21.423041234Z\",\n \"endTime\": \"2026-09-20T18:58:28.878805957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sustfjvol3npjlb5trmhmvn0qgrv50a still exists (deletion dispatched at 2026-09-20T18:34:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00138a198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-x79rddkrff4p\" to finish deleting: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5321d526-4d45-49af-b781-508e5635a670\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5321d526-4d45-49af-b781-508e5635a670\",\n \"name\": \"5321d526-4d45-49af-b781-508e5635a670\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T18:34:21.423041234Z\",\n \"endTime\": \"2026-09-20T18:58:28.878805957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sustfjvol3npjlb5trmhmvn0qgrv50a still exists (deletion dispatched at 2026-09-20T18:34:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6c3ddb6263f49fc3f8d369e07efe10154c44e91bf4381fdf9ef73e1170f109cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"6bc7534690e6bb03e8cc5f05e1f2bfee87b13f961ed915d568dd8e6b92f0e0f2","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a56a60\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8wclt4mlsd89\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc19e435-ddc8-468a-83ba-b4ec238ff165\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc19e435-ddc8-468a-83ba-b4ec238ff165\",\n \"name\": \"bc19e435-ddc8-468a-83ba-b4ec238ff165\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:58.826266654Z\",\n \"endTime\": \"2026-09-20T18:19:00.407536942Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a56a60\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8wclt4mlsd89\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc19e435-ddc8-468a-83ba-b4ec238ff165\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc19e435-ddc8-468a-83ba-b4ec238ff165\",\n \"name\": \"bc19e435-ddc8-468a-83ba-b4ec238ff165\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:58.826266654Z\",\n \"endTime\": \"2026-09-20T18:19:00.407536942Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"75e647cb74cc1b5f69e5e1695080def5d27fc40d0a9c28a957bd70da7446fa8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"aea772e72e8fe85891b43563346768f4449021077f900a91ce91ffe2eb62aef3","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00181a220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-xth4kfdgl9qd\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84362293-473a-4529-8480-825894137a63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84362293-473a-4529-8480-825894137a63\",\n \"name\": \"84362293-473a-4529-8480-825894137a63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:49.637399572Z\",\n \"endTime\": \"2026-09-20T18:18:50.472292068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00181a220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-xth4kfdgl9qd\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84362293-473a-4529-8480-825894137a63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84362293-473a-4529-8480-825894137a63\",\n \"name\": \"84362293-473a-4529-8480-825894137a63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:49.637399572Z\",\n \"endTime\": \"2026-09-20T18:18:50.472292068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"98057eac6d67991ec7317fadc3faa2bb273575ab3c78370d97be2119ff2eabff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"alert","signature_id":"8da3d9c87ac6e15f84a876a8ec0eb6d135540c742388bef66277cd5947ec5e3b","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 20 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:28:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:26:53Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-20T18:10:51Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-20T18:10:51Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-20T18:10:51Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-20T18:10:52Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"9c766dd07bf1d4cf3ad48907cf7c9e72d78833a1502e0375573da2c5e97ae1dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"c4e33f88c11b927f53f19ef5f88b383cbdc6921ffa0d1b5ed16a016693d69ee1","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000888600\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-d7ffrxgdklfr\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54b46b89-b4a9-4ae9-b551-78a76e071ebf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54b46b89-b4a9-4ae9-b551-78a76e071ebf\",\n \"name\": \"54b46b89-b4a9-4ae9-b551-78a76e071ebf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:02.573105706Z\",\n \"endTime\": \"2026-09-20T18:19:10.459388953Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000888600\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-d7ffrxgdklfr\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54b46b89-b4a9-4ae9-b551-78a76e071ebf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54b46b89-b4a9-4ae9-b551-78a76e071ebf\",\n \"name\": \"54b46b89-b4a9-4ae9-b551-78a76e071ebf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:02.573105706Z\",\n \"endTime\": \"2026-09-20T18:19:10.459388953Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9d5a5b5375c0faa5ae269008f3d10cb24a7bf5d7768776210f273bcc750e9a7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"a682bdcc6ac5308107646bacc01ed06ff73822f3817c828a7af6eeb2d50dbb37","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097f060\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-mpqhctkn8kzf\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f0788d5f-c399-4ffb-8320-8f18588385ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f0788d5f-c399-4ffb-8320-8f18588385ec\",\n \"name\": \"f0788d5f-c399-4ffb-8320-8f18588385ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:48.534799023Z\",\n \"endTime\": \"2026-09-20T18:18:50.467299699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097f060\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-mpqhctkn8kzf\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f0788d5f-c399-4ffb-8320-8f18588385ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f0788d5f-c399-4ffb-8320-8f18588385ec\",\n \"name\": \"f0788d5f-c399-4ffb-8320-8f18588385ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:48.534799023Z\",\n \"endTime\": \"2026-09-20T18:18:50.467299699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a6907afe93b489df04a82546f5045cc945e5154bbf8ca037a7c533d981852c83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"f37d1bd86a2dec4c46be9ce8fb4c03ed08425d29a0d32e20dd962b464a680228","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000696760\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-lzkpz5ftlkll\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\",\n \"name\": \"83245618-9845-4872-97a3-4ee524027d39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:02.101717721Z\",\n \"endTime\": \"2026-09-20T18:19:10.391207338Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000696760\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-lzkpz5ftlkll\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\",\n \"name\": \"83245618-9845-4872-97a3-4ee524027d39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:02.101717721Z\",\n \"endTime\": \"2026-09-20T18:19:10.391207338Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"44fc5c170ee5969da686dbaeb61a39c91b4e9f01cf7a4324b16151d7f8a7811a","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b688759f4d2ab67c229ce65b6ddfb3031692a5c8bd4c2a7870e657c2cb70a837","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"7486390772235b7b4ad872f03600862b7205a72c98df5cd28e716390fedf267b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-gns5q5 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ec160\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-gns5q5: failed waiting for cluster=\"np-version-upgrade-cluster-gns5q5\" in resourcegroup=\"rg-np-version-upgrade-gns5q5-hq6mhxkh7bdl\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4673ce57-b0e5-4848-88c0-86c48e4b213b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4673ce57-b0e5-4848-88c0-86c48e4b213b\",\n \"name\": \"4673ce57-b0e5-4848-88c0-86c48e4b213b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:48.158843811Z\",\n \"endTime\": \"2026-09-20T18:18:50.501392807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-gns5q5 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ec160\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-gns5q5: failed waiting for cluster=\"np-version-upgrade-cluster-gns5q5\" in resourcegroup=\"rg-np-version-upgrade-gns5q5-hq6mhxkh7bdl\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4673ce57-b0e5-4848-88c0-86c48e4b213b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4673ce57-b0e5-4848-88c0-86c48e4b213b\",\n \"name\": \"4673ce57-b0e5-4848-88c0-86c48e4b213b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:48.158843811Z\",\n \"endTime\": \"2026-09-20T18:18:50.501392807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c057079edbc6dcf512a4ce5da52ef44c89587a9d7146e3535615d6a7b95ab7c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"alert","signature_id":"8389beb2f2127c28d1c407bb2631de05104a4b0dfdbf0a89c145c62d14049937","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 16 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-20T18:24:37Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-20T18:24:37Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2susts60dnkbabh5ft9bj4gfhtr5ubsk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2susts60dnkbabh5ft9bj4gfhtr5ubsk/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-20T18:26:36Z\n Ended: 2026-09-20T18:32:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2susu70nvmv9oua374lkat5p6me0e8uj\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2susu70nvmv9oua374lkat5p6me0e8uj/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","44fc5c170ee5969da686dbaeb61a39c91b4e9f01cf7a4324b16151d7f8a7811a","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101718617941872640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T17:02:21Z","started_at":"2026-09-20T17:02:21Z","completed_at":"2026-09-20T17:39:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T17:48:53.075215375Z","timing_metadata_checked_at":"2026-09-20T17:48:53.075215375Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T17:48:53.079094461Z","region_metadata_checked_at":"2026-09-20T17:48:53.079094461Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c586a78806d92ee0c5fb78f36f018eadb2ffc0c4235cb63fb6c858a453e907eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101718617941872640","occurred_at":"2026-09-20T17:02:21Z","failed_at":"provision","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-20T17:31:48.163Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-20T17:31:48.670Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-20T17:31:49.380Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1872640-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6\ntime=2026-09-20T17:33:49.754Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.Resources/deployments/113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6/operationStatuses/08584116813764412291\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-post\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"NotFound\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e07cda02c84cd6ea2552fbcc5179ebfd519e66665b62547f1ea88bc8136bc936","failure_pattern":"ERROR CODE: DeploymentFailed; detail code NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e07cda02c84cd6ea2552fbcc5179ebfd519e66665b62547f1ea88bc8136bc936"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101715422586867712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7033,"pr_state":"closed","pr_sha":"7dc06a6c4270438febda85df95c9bddffa9eb2c6","final_merged_sha":"ef821496d20065a3db001282a41d83827d994051","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T16:49:39Z","started_at":"2026-09-20T16:49:39Z","completed_at":"2026-09-20T19:06:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T19:12:48.083790668Z","timing_metadata_checked_at":"2026-09-20T19:12:48.083790668Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T19:12:48.090248749Z","region_metadata_checked_at":"2026-09-20T19:12:48.090248749Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"17a14a0957a4b84810803dbeea78325f1fe2cfd61a0d9d244ccc1397a6e55099","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101715422586867712","occurred_at":"2026-09-20T16:49:39Z","failed_at":"alert","signature_id":"a1c099010ae016739013a1e87affb32949a8f1160e4fb45e709b3366b93a0cff","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T18:17:23Z\n Ended: 2026-09-20T18:33:23Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6867712-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j6867712-svc).\nFiring 2:\n State: Resolved\n Started: 2026-09-20T18:17:23Z\n Ended: 2026-09-20T18:32:24Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6867712-svc\", component=\"frontend\", method=\"delete\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j6867712-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"66114193321438d473fff92dbe41e69829ce45a7dd02d3c9116ac30b8d4e7ab2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101715422586867712","occurred_at":"2026-09-20T16:49:39Z","failed_at":"e2e","signature_id":"c01cceb55dbabd6acf2c61e2aecc8f6bbf9895f9ca156a63db239382788d9d1e","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000eb2060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-8mmkqkc7dspl\" to finish deleting: GET https://rp.j6867712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f73e80f-46b4-48bf-9496-30e3a271cd59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f73e80f-46b4-48bf-9496-30e3a271cd59\",\n \"name\": \"9f73e80f-46b4-48bf-9496-30e3a271cd59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T18:26:50.030271411Z\",\n \"endTime\": \"2026-09-20T18:50:50.442666544Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suspvnhq9bs89art5mn22rcv6o4gr53 still exists (deletion dispatched at 2026-09-20T18:26:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000eb2060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-8mmkqkc7dspl\" to finish deleting: GET https://rp.j6867712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f73e80f-46b4-48bf-9496-30e3a271cd59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f73e80f-46b4-48bf-9496-30e3a271cd59\",\n \"name\": \"9f73e80f-46b4-48bf-9496-30e3a271cd59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T18:26:50.030271411Z\",\n \"endTime\": \"2026-09-20T18:50:50.442666544Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suspvnhq9bs89art5mn22rcv6o4gr53 still exists (deletion dispatched at 2026-09-20T18:26:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101708991846420480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7032,"pr_state":"closed","pr_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","final_merged_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T16:24:06Z","started_at":"2026-09-20T16:24:06Z","completed_at":"2026-09-20T18:29:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T18:41:06.241339209Z","timing_metadata_checked_at":"2026-09-20T18:41:06.241339209Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T18:41:06.245039896Z","region_metadata_checked_at":"2026-09-20T18:41:06.245039896Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"04189d0f5e20a17650ae09a53aee20f93e067674440838af21ebe94baf5efb16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101708991846420480","occurred_at":"2026-09-20T16:24:06Z","failed_at":"e2e","signature_id":"7a7b14873ebc7e928a8df3faa64a26b8a0c2ace4736d3d613f6a89c5b2c7ebba","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00116c378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-9t8q6g\" in resourcegroup=\"rg-np-version-upgrade-9t8q6g-d8vnpqdkzzdd\" to finish deleting: GET https://rp.j6420480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\",\n \"name\": \"a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:41:51.558513909Z\",\n \"endTime\": \"2026-09-20T18:05:54.797697774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus7kr9oc70h1qkt3fma8jukokvb4fq still exists (deletion dispatched at 2026-09-20T17:41:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00116c378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-9t8q6g\" in resourcegroup=\"rg-np-version-upgrade-9t8q6g-d8vnpqdkzzdd\" to finish deleting: GET https://rp.j6420480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\",\n \"name\": \"a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:41:51.558513909Z\",\n \"endTime\": \"2026-09-20T18:05:54.797697774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus7kr9oc70h1qkt3fma8jukokvb4fq still exists (deletion dispatched at 2026-09-20T17:41:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ba7095bc9215bcd43f1ce1c9be66b542a2bb9543fff4f416ae7f7cd986a2f3b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101708991846420480","occurred_at":"2026-09-20T16:24:06Z","failed_at":"e2e","signature_id":"f680e9c6bf353ff7e3737b28ae5ee7746498d1917182a5b8a67f475e211714a3","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012d8150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-77nnw8xhpg6s\" to finish deleting: GET https://rp.j6420480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83c11edb-cb7a-458f-8d8e-0fa679b34fd8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83c11edb-cb7a-458f-8d8e-0fa679b34fd8\",\n \"name\": \"83c11edb-cb7a-458f-8d8e-0fa679b34fd8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:48:44.753307034Z\",\n \"endTime\": \"2026-09-20T18:12:50.043857017Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus7ldf7cgmduvr2ibfvtv6auch4u6v still exists (deletion dispatched at 2026-09-20T17:48:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012d8150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-77nnw8xhpg6s\" to finish deleting: GET https://rp.j6420480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83c11edb-cb7a-458f-8d8e-0fa679b34fd8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83c11edb-cb7a-458f-8d8e-0fa679b34fd8\",\n \"name\": \"83c11edb-cb7a-458f-8d8e-0fa679b34fd8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:48:44.753307034Z\",\n \"endTime\": \"2026-09-20T18:12:50.043857017Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus7ldf7cgmduvr2ibfvtv6auch4u6v still exists (deletion dispatched at 2026-09-20T17:48:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7030,"pr_state":"closed","pr_sha":"4dc2c443d9d2f9fa2a598585253f8231c7ddf46c","final_merged_sha":"50f9710d6eb949ad5cf7c325f7c4d6cce9083ddf","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T16:10:57Z","started_at":"2026-09-20T16:10:57Z","completed_at":"2026-09-20T18:22:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T18:30:39.734701475Z","timing_metadata_checked_at":"2026-09-20T18:30:39.734701475Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T18:30:39.738181764Z","region_metadata_checked_at":"2026-09-20T18:30:39.738181764Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"510d5decb7dbc656e3bab754252f21edb285f50b483628007149b22b8f1c8352","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T17:13:02Z\n Ended: 2026-09-20T17:29:03Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5465216-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"6b7d8fb73aaeedccd83485a3889248f07a8bebc749a873d77b78ee01c97cf23b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","failed_at":"e2e","signature_id":"b369fa1435784bfd120c7576bed94a6e4a39fcf725052260c677eef0e62863e5","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010ca0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-v62fgp56rkgl\" to finish deleting: GET https://rp.j5465216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00507be0-8c9c-4d23-b9ba-c8c11bbcd318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00507be0-8c9c-4d23-b9ba-c8c11bbcd318\",\n \"name\": \"00507be0-8c9c-4d23-b9ba-c8c11bbcd318\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:32:28.39845259Z\",\n \"endTime\": \"2026-09-20T17:56:32.271542783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus5u9v5u2tuv9eqg0al8679ob1cvtj still exists (deletion dispatched at 2026-09-20T17:32:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010ca0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-v62fgp56rkgl\" to finish deleting: GET https://rp.j5465216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00507be0-8c9c-4d23-b9ba-c8c11bbcd318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00507be0-8c9c-4d23-b9ba-c8c11bbcd318\",\n \"name\": \"00507be0-8c9c-4d23-b9ba-c8c11bbcd318\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:32:28.39845259Z\",\n \"endTime\": \"2026-09-20T17:56:32.271542783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus5u9v5u2tuv9eqg0al8679ob1cvtj still exists (deletion dispatched at 2026-09-20T17:32:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a789f4ea74f6d398875de4e36ef0c56467e2d8149e7a77e09db5818d8f340316","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","failed_at":"e2e","signature_id":"2eff4fde8ffd9f3247fb66660611f250c208eb3abb98228934a36763af0a55d7","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000c3c360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-s96znfczgxhl\" to finish deleting: GET https://rp.j5465216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b6ea12d-1868-4480-9201-fbbeb535fe47\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b6ea12d-1868-4480-9201-fbbeb535fe47\",\n \"name\": \"1b6ea12d-1868-4480-9201-fbbeb535fe47\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:40:48.648177125Z\",\n \"endTime\": \"2026-09-20T18:04:52.302253283Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus4q0ga6s9rpgj08rupc4dvqq0lr7j still exists (deletion dispatched at 2026-09-20T17:40:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000c3c360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-s96znfczgxhl\" to finish deleting: GET https://rp.j5465216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b6ea12d-1868-4480-9201-fbbeb535fe47\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b6ea12d-1868-4480-9201-fbbeb535fe47\",\n \"name\": \"1b6ea12d-1868-4480-9201-fbbeb535fe47\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:40:48.648177125Z\",\n \"endTime\": \"2026-09-20T18:04:52.302253283Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus4q0ga6s9rpgj08rupc4dvqq0lr7j still exists (deletion dispatched at 2026-09-20T17:40:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7029/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101687697306292224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7029,"pr_state":"closed","pr_sha":"0ddd8cc7d917131bb10132dfa6c0291c472c1b5d","final_merged_sha":"0ddd8cc7d917131bb10132dfa6c0291c472c1b5d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T14:59:29Z","started_at":"2026-09-20T14:59:29Z","completed_at":"2026-09-20T16:54:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T17:06:49.068278175Z","timing_metadata_checked_at":"2026-09-20T17:06:49.068278175Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T17:06:49.072180963Z","region_metadata_checked_at":"2026-09-20T17:06:49.072180963Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101683713959006208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T14:43:39Z","started_at":"2026-09-20T14:43:39Z","completed_at":"2026-09-20T17:08:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T17:17:11.794656569Z","timing_metadata_checked_at":"2026-09-20T17:17:11.794656569Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T17:17:11.798326057Z","region_metadata_checked_at":"2026-09-20T17:17:11.798326057Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"41ea2612881829bb7dc4582ecca40194a0f67e5781273ef7f1b3bad345f8aa3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101683713959006208","occurred_at":"2026-09-20T14:43:39Z","failed_at":"alert","signature_id":"19e557aaac4e676680853d6de094c41a779d1dca118ed39692cb9878dfd677f7","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T15:44:51Z\n Ended: 2026-09-20T16:04:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9006208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2suqt99oepq62cp9g8imp789kb8cucv2-u7z6v1b8p9z3l2n\", pod=\"router-75ddb8448b-mcjrz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2suqt99oepq62cp9g8imp789kb8cucv2-u7z6v1b8p9z3l2n/router-75ddb8448b-mcjrz has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-20T15:45:49Z\n Ended: 2026-09-20T16:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9006208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2suqteqhtuf74ce2qrveg0u8hgjr0p5r-e7c9f2r2o1e9v6w\", pod=\"router-7d46777b99-llb2x\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2suqteqhtuf74ce2qrveg0u8hgjr0p5r-e7c9f2r2o1e9v6w/router-7d46777b99-llb2x has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-20T15:45:49Z\n Ended: 2026-09-20T16:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9006208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2suqudsrpaqa5rkctesukjsg27o2l1a8-j6b1k8a5z0w9b4r\", pod=\"router-68967d6b9b-mjwwc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2suqudsrpaqa5rkctesukjsg27o2l1a8-j6b1k8a5z0w9b4r/router-68967d6b9b-mjwwc has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-20T15:46:51Z\n Ended: 2026-09-20T16:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9006208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2suqtr0pu725fj3uus9447oj3b8r0ga6-d9e8m6u9f5g8f7e\", pod=\"router-bf89df67f-dc5rn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2suqtr0pu725fj3uus9447oj3b8r0ga6-d9e8m6u9f5g8f7e/router-bf89df67f-dc5rn has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c972a5a828f0470963a3b4b90d8a4fd80eb05f65468d3843254d0b563dbaa2af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101683713959006208","occurred_at":"2026-09-20T14:43:39Z","failed_at":"alert","signature_id":"fb812e0e230afdfaf46a5827aae975ba58d618c82618d4f8ed79f5468297e6a0","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-20T16:39:02Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9006208-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vbl5w\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-vbl5w (namespace arobit) on cluster ci01-j9006208-svc has averaged 126.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T14:02:20Z","started_at":"2026-09-20T14:02:20Z","completed_at":"2026-09-20T15:57:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T16:03:53.495620959Z","timing_metadata_checked_at":"2026-09-20T16:03:53.495620959Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T16:03:53.499134818Z","region_metadata_checked_at":"2026-09-20T16:03:53.499134818Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"41c32a3a2824f046723c236a74b45faee8f8ebd5459b2096613a68a4dcbdbfbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","failed_at":"e2e","signature_id":"21b7ce7b54d77195fc617e821329b90e750c10192c6be2fdd3f4089b6662713d","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cbe240\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mhtls7w2h6hh\" to finish creating: GET https://rp.j6358528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1160ee4-c1b1-4f87-a844-83d0551e46a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1160ee4-c1b1-4f87-a844-83d0551e46a1\",\n \"name\": \"e1160ee4-c1b1-4f87-a844-83d0551e46a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T14:49:03.140717865Z\",\n \"endTime\": \"2026-09-20T15:18:05.481184474Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cbe240\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mhtls7w2h6hh\" to finish creating: GET https://rp.j6358528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1160ee4-c1b1-4f87-a844-83d0551e46a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1160ee4-c1b1-4f87-a844-83d0551e46a1\",\n \"name\": \"e1160ee4-c1b1-4f87-a844-83d0551e46a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T14:49:03.140717865Z\",\n \"endTime\": \"2026-09-20T15:18:05.481184474Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d799d1f5a1cc22da71d3398ecf519fc25a4c93ad679c4139fdfc45d659168f70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","failed_at":"alert","signature_id":"e66e5d2206cbbff91631193f79b282c5f04f3b6efea313eb1c9fc43af3b6f8ad","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T15:11:10Z\n Ended: 2026-09-20T15:36:05Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6358528-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.107:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6358528/providers/microsoft.monitor/accounts/hcps-j6358528\", namespace=\"ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-x5q5k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"e23140e3f627c654ababf7eba526daf0cc71c26b8e8cee26d7b7ae12c1588d9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","failed_at":"alert","signature_id":"84036f94843f01e49f13138f55d77ae74e49b96060ac2cc402fa01f03e462128","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T15:26:04Z\n Ended: 2026-09-20T15:41:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6358528-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.107:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6358528/providers/microsoft.monitor/accounts/hcps-j6358528\", namespace=\"ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-x5q5k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673266119708672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7025,"pr_state":"closed","pr_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","final_merged_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T14:02:08Z","started_at":"2026-09-20T14:02:08Z","completed_at":"2026-09-20T15:59:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T16:14:44.390494098Z","timing_metadata_checked_at":"2026-09-20T16:14:44.390494098Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T16:14:44.393971187Z","region_metadata_checked_at":"2026-09-20T16:14:44.393971187Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101670587393904640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7026,"pr_state":"closed","pr_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","final_merged_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T13:51:29Z","started_at":"2026-09-20T13:51:29Z","completed_at":"2026-09-20T16:01:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T16:14:44.493587366Z","timing_metadata_checked_at":"2026-09-20T16:14:44.493587366Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T16:14:44.496047758Z","region_metadata_checked_at":"2026-09-20T16:14:44.496047758Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f15241c0a1f51c5285e92def8ad6c40e26a0cfd063837b73ec858ce9a07a2fff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101670587393904640","occurred_at":"2026-09-20T13:51:29Z","failed_at":"e2e","signature_id":"10316e5a87c9726ae96606bd5b16607ddf7b0d400689c8e8c0e050115901ad56","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017da108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7v2dcntmwrc6\" to finish deleting: GET https://rp.j3904640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05e626d4-f861-4914-be3d-d9945e36c566\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05e626d4-f861-4914-be3d-d9945e36c566\",\n \"name\": \"05e626d4-f861-4914-be3d-d9945e36c566\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T15:22:08.248375345Z\",\n \"endTime\": \"2026-09-20T15:46:08.421611372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suq3e6ic386bksmrgs696k8ca3rriju still exists (deletion dispatched at 2026-09-20T15:22:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017da108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7v2dcntmwrc6\" to finish deleting: GET https://rp.j3904640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05e626d4-f861-4914-be3d-d9945e36c566\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05e626d4-f861-4914-be3d-d9945e36c566\",\n \"name\": \"05e626d4-f861-4914-be3d-d9945e36c566\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T15:22:08.248375345Z\",\n \"endTime\": \"2026-09-20T15:46:08.421611372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suq3e6ic386bksmrgs696k8ca3rriju still exists (deletion dispatched at 2026-09-20T15:22:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101647291294158848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7025,"pr_state":"closed","pr_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","final_merged_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T12:18:55Z","started_at":"2026-09-20T12:18:55Z","completed_at":"2026-09-20T13:57:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T14:07:52.394475216Z","timing_metadata_checked_at":"2026-09-20T14:07:52.394475216Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T14:07:52.397835104Z","region_metadata_checked_at":"2026-09-20T14:07:52.397835104Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5d11569701ec4fedd520cab68d1579802c7157ba87b8355b91ce4c44dc6f1a35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101647291294158848","occurred_at":"2026-09-20T12:18:55Z","failed_at":"e2e","signature_id":"e0ec91f3cf6a62bae5b4c3d56509ed71730ec1128a9898d0313dd55a382d4fa9","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:184]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ca8a20\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:184]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ca8a20\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","failure_pattern":"UpdateHCPCluster20260901 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643653972758528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7030,"pr_state":"closed","pr_sha":"4dc2c443d9d2f9fa2a598585253f8231c7ddf46c","final_merged_sha":"50f9710d6eb949ad5cf7c325f7c4d6cce9083ddf","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T12:04:28Z","started_at":"2026-09-20T12:04:28Z","completed_at":"2026-09-20T14:06:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T14:18:22.536443847Z","timing_metadata_checked_at":"2026-09-20T14:18:22.536443847Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T14:18:22.539587236Z","region_metadata_checked_at":"2026-09-20T14:18:22.539587236Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"34e9c57ed0be8aa68a4d9c94a38d30de4f2709abb5ea2f95778c521dc5c58c4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643653972758528","occurred_at":"2026-09-20T12:04:28Z","failed_at":"alert","signature_id":"f483622241db14070571766c5ea34d86fd6f53d96057f1964f32d162917e2451","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T13:27:17Z\n Ended: 2026-09-20T13:41:19Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j2758528-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2758528-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7029/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643457066962944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7029,"pr_state":"closed","pr_sha":"0ddd8cc7d917131bb10132dfa6c0291c472c1b5d","final_merged_sha":"0ddd8cc7d917131bb10132dfa6c0291c472c1b5d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T12:03:41Z","started_at":"2026-09-20T12:03:41Z","completed_at":"2026-09-20T14:21:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T14:28:56.140128041Z","timing_metadata_checked_at":"2026-09-20T14:28:56.140128041Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T14:28:56.14324203Z","region_metadata_checked_at":"2026-09-20T14:28:56.14324203Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"45895f26687a3e4c5bbb69605b2f68bdd724847a15f09cbd57f0cfc433e842e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7029/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643457066962944","occurred_at":"2026-09-20T12:03:41Z","failed_at":"e2e","signature_id":"5fb2a2918cf998e33798a60ea71579a7a8d626d7c294f1e7d5559e93c8cb6517","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c380d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xw2mr9c7sf7t\" to finish deleting: GET https://rp.j6962944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13baf428-ed80-4931-a4b3-8398b975e18c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13baf428-ed80-4931-a4b3-8398b975e18c\",\n \"name\": \"13baf428-ed80-4931-a4b3-8398b975e18c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T13:41:50.034520019Z\",\n \"endTime\": \"2026-09-20T14:05:52.42378958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suoj1308cgeas4uvag9eqertadh2q3h still exists (deletion dispatched at 2026-09-20T13:41:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c380d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xw2mr9c7sf7t\" to finish deleting: GET https://rp.j6962944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13baf428-ed80-4931-a4b3-8398b975e18c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13baf428-ed80-4931-a4b3-8398b975e18c\",\n \"name\": \"13baf428-ed80-4931-a4b3-8398b975e18c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T13:41:50.034520019Z\",\n \"endTime\": \"2026-09-20T14:05:52.42378958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suoj1308cgeas4uvag9eqertadh2q3h still exists (deletion dispatched at 2026-09-20T13:41:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T12:02:36Z","started_at":"2026-09-20T12:02:36Z","completed_at":"2026-09-20T13:40:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T13:46:25.016731887Z","timing_metadata_checked_at":"2026-09-20T13:46:25.016731887Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T13:46:25.019790678Z","region_metadata_checked_at":"2026-09-20T13:46:25.019790678Z"},"failed_at":["e2e"],"failed_test_count":21,"occurrences":[{"row_id":"29ce18dea7d175c68db3a3ef5f84e9381c6919f529a5085f6de3dd2209e22beb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"91a719e5ea3aea8b9b0e188fedf3ccfcff48a4a62dc5fc1afe26e222804d6227","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-5-0-7gj726\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001893500\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-zstream-5-0-7gj726-5thxzf\" in resourcegroup=\"rg-zstream-upgrade-5-0-nvfh99dzvbhb\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-5-0-nvfh99dzvbhb/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-5-0-7gj726-5thxzf/operationStatuses/08584116992455805325\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-5-0-7gj726\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001893500\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-zstream-5-0-7gj726-5thxzf\" in resourcegroup=\"rg-zstream-upgrade-5-0-nvfh99dzvbhb\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-5-0-nvfh99dzvbhb/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-5-0-7gj726-5thxzf/operationStatuses/08584116992455805325\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"34810f1b5359320d18ca47e793c3086eaaf20eb9e286ce83d65e9dd26aa6edf2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"75b9233e5419e103c738cc032e9a0d4380e13a4936bd5bad2e51413dae0bf26f","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002b80c0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-arm64-vm-hcp-cluster-q2fmvf\" in resourcegroup=\"arm64-vm-cluster-6ct9s56bgsnd\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-6ct9s56bgsnd/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-q2fmvf/operationStatuses/08584116992449832602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002b80c0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-arm64-vm-hcp-cluster-q2fmvf\" in resourcegroup=\"arm64-vm-cluster-6ct9s56bgsnd\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-6ct9s56bgsnd/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-q2fmvf/operationStatuses/08584116992449832602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3d854a444148a1fe3d93332ca05ff812f13dce922681796f90959e3bc583aa07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"9db4a31a545e08b96a9f9b2a51e0ec6fa079b7393b1270544cac4af6f6246fd5","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a714e0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-idms-e2e-hcp-cluster-nh4pbh\" in resourcegroup=\"idms-l5zltlxr72bw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/idms-l5zltlxr72bw/providers/Microsoft.Resources/deployments/customer-infra-idms-e2e-hcp-cluster-nh4pbh/operationStatuses/08584116992375018318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a714e0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-idms-e2e-hcp-cluster-nh4pbh\" in resourcegroup=\"idms-l5zltlxr72bw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/idms-l5zltlxr72bw/providers/Microsoft.Resources/deployments/customer-infra-idms-e2e-hcp-cluster-nh4pbh/operationStatuses/08584116992375018318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5ae5159e14bf52b4e12cef329e6c3c98b41e6c39ed772fc447c1107dfa2c619a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"b9513cbbafa37244dbcd3207b8b1836ea824ef6ed8a8fa6ee5005da287c245c1","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015224a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-dg-minor-97cpdn-97nfmz\" in resourcegroup=\"rg-np-dg-minor-97cpdn-jwdnmqz4h5k5\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-dg-minor-97cpdn-jwdnmqz4h5k5/providers/Microsoft.Resources/deployments/customer-infra-np-dg-minor-97cpdn-97nfmz/operationStatuses/08584116992296089268\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015224a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-dg-minor-97cpdn-97nfmz\" in resourcegroup=\"rg-np-dg-minor-97cpdn-jwdnmqz4h5k5\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-dg-minor-97cpdn-jwdnmqz4h5k5/providers/Microsoft.Resources/deployments/customer-infra-np-dg-minor-97cpdn-97nfmz/operationStatuses/08584116992296089268\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"68446b8a5e2ce286646f100ed7f22e624a7e85a5a990e79c0e247c0429e70fe5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"bd221a69442a6d79a216e48307db3abafc2724516fd0d3101335d3d4bb683a2d","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0aa80\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-version-upgrade-cluster-5sgmvt-wh2fh7\" in resourcegroup=\"rg-np-version-upgrade-5sgmvt-ddwpmwwlkbp8\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-5sgmvt-ddwpmwwlkbp8/providers/Microsoft.Resources/deployments/customer-infra-np-version-upgrade-cluster-5sgmvt-wh2fh7/operationStatuses/08584116992292355490\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0aa80\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-version-upgrade-cluster-5sgmvt-wh2fh7\" in resourcegroup=\"rg-np-version-upgrade-5sgmvt-ddwpmwwlkbp8\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-5sgmvt-ddwpmwwlkbp8/providers/Microsoft.Resources/deployments/customer-infra-np-version-upgrade-cluster-5sgmvt-wh2fh7/operationStatuses/08584116992292355490\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6a78de9e602b3241a40519368cf2bd66924ae7f395ee9425783080a028598d67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"1a79c558038cb73ff72359e020fc0d6f9f740e31cbd0af1daebaf91ad6c65fa3","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:76]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f02d60\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kas-m75gws\" in resourcegroup=\"private-kas-s98lp67jv589\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-kas-s98lp67jv589/providers/Microsoft.Resources/deployments/customer-infra-private-kas-m75gws/operationStatuses/08584116992457285882\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:76]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f02d60\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kas-m75gws\" in resourcegroup=\"private-kas-s98lp67jv589\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-kas-s98lp67jv589/providers/Microsoft.Resources/deployments/customer-infra-private-kas-m75gws/operationStatuses/08584116992457285882\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6bd3fc1c3ef6e8fda466478243f384ca27bd1743540d8d7b525c69212e27b9e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"454410db4b0c9c7e89e9e307d8cabd3be3a5cc6c620a919683872059242695a3","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:72]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb0540\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-fips-enabled-cluster-d9t886\" in resourcegroup=\"fips-enabled-rvgn6tk7vl7q\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/fips-enabled-rvgn6tk7vl7q/providers/Microsoft.Resources/deployments/customer-infra-fips-enabled-cluster-d9t886/operationStatuses/08584116992455136747\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:72]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb0540\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-fips-enabled-cluster-d9t886\" in resourcegroup=\"fips-enabled-rvgn6tk7vl7q\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/fips-enabled-rvgn6tk7vl7q/providers/Microsoft.Resources/deployments/customer-infra-fips-enabled-cluster-d9t886/operationStatuses/08584116992455136747\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6c240beaa637653d1dfc84c9fe493b860a14ef71913a42d125539a3d36adb8c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"56478c58658652b4bc1945a590ac05ad32b42a6f22c2e138039cb7ee3c4cbfac","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2f940\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-7mb4tc\" in resourcegroup=\"external-auth-cluster-bhfp8rbccvvf\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-bhfp8rbccvvf/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-7mb4tc/operationStatuses/08584116992371740355\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2f940\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-7mb4tc\" in resourcegroup=\"external-auth-cluster-bhfp8rbccvvf\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-bhfp8rbccvvf/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-7mb4tc/operationStatuses/08584116992371740355\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6cbb32f2d070654308a2db71c06b3791edd01d3792fdd3c8f1af09b7fa8b4be7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"a186884d8b4f85878973e1e8bb26766fcd36f0f99234916982d31c2386197c2a","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:74]: failed to create customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d89980\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-pause-bkp-cluster-9zrvd8\" in resourcegroup=\"pause-bkp-e2e-5kfj7kdvg7fr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/pause-bkp-e2e-5kfj7kdvg7fr/providers/Microsoft.Resources/deployments/customer-infra-pause-bkp-cluster-9zrvd8/operationStatuses/08584116992375377229\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:74]: failed to create customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d89980\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-pause-bkp-cluster-9zrvd8\" in resourcegroup=\"pause-bkp-e2e-5kfj7kdvg7fr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/pause-bkp-e2e-5kfj7kdvg7fr/providers/Microsoft.Resources/deployments/customer-infra-pause-bkp-cluster-9zrvd8/operationStatuses/08584116992375377229\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7a9b88fe228b0082f576b5fb455ed426081893406605762f234f792888ba1a66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"42440a2d7bb764d5edcdcdcd65e32a762540470eaa9f7ce1559f0c2a375c81a0","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b32200\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-autoscale-cluster-gz6j7m\" in resourcegroup=\"np-autoscaling-d6nt8sqhc4v2\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-d6nt8sqhc4v2/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-gz6j7m/operationStatuses/08584116992287759958\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b32200\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-autoscale-cluster-gz6j7m\" in resourcegroup=\"np-autoscaling-d6nt8sqhc4v2\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-d6nt8sqhc4v2/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-gz6j7m/operationStatuses/08584116992287759958\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7ba761628929a594d4d994b456f8ce091cb1625060cc24b8405a3a1a80d31e25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"0899e170df3dbff91a3d93d67607d26e9d9e1d55e607106e4712f92b087c03fd","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0027fddb0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0027fddb0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"807a4aa3e07746639902538cdb3c7f789ffe3f005cda5c48a49cb81c2f622ac3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"81b579506962c7694460b4e0ba3269306f93397ebe18eb85545cae4066854205","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001782300\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-version-upgrade-cluster-x7b8c9-g9hng6\" in resourcegroup=\"rg-np-version-upgrade-x7b8c9-78csh8btgppk\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-x7b8c9-78csh8btgppk/providers/Microsoft.Resources/deployments/customer-infra-np-version-upgrade-cluster-x7b8c9-g9hng6/operationStatuses/08584116992297707311\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001782300\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-version-upgrade-cluster-x7b8c9-g9hng6\" in resourcegroup=\"rg-np-version-upgrade-x7b8c9-78csh8btgppk\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-x7b8c9-78csh8btgppk/providers/Microsoft.Resources/deployments/customer-infra-np-version-upgrade-cluster-x7b8c9-g9hng6/operationStatuses/08584116992297707311\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"87bfff40d4a206f858615acd48a139316c21612c4b6785262b6be28bb3d66030","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"d798f3f3a8a081a020bf7521957c622331f9901d07425be35f0617500d755d67","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-8n7lrz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00096f0a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-22-8n7lrz-jq5k6h\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v9fgb6qclpxh\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-22-v9fgb6qclpxh/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-22-8n7lrz-jq5k6h/operationStatuses/08584116992380864708\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-8n7lrz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00096f0a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-22-8n7lrz-jq5k6h\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v9fgb6qclpxh\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-22-v9fgb6qclpxh/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-22-8n7lrz-jq5k6h/operationStatuses/08584116992380864708\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"97e5e7db391a5ea3a14515531b6e53e2f94b88bf52da9410a7e2c61c102543d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"73c41b364dee2fd3eb27b4ccaa57fb601deed774cfcc5111654a86e31aa1c0ef","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:83]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fce560\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cilium-cluster-779vpn\" in resourcegroup=\"complex-cilium-kv-hj2fprxw8lj6\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/complex-cilium-kv-hj2fprxw8lj6/providers/Microsoft.Resources/deployments/customer-infra-cilium-cluster-779vpn/operationStatuses/08584116992458344159\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:83]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fce560\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cilium-cluster-779vpn\" in resourcegroup=\"complex-cilium-kv-hj2fprxw8lj6\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/complex-cilium-kv-hj2fprxw8lj6/providers/Microsoft.Resources/deployments/customer-infra-cilium-cluster-779vpn/operationStatuses/08584116992458344159\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a6086b66362884bb02fb90142aa5fa84d59bd590d924f79597287be439d5e48b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"5d0ca65ae867bd15279982b3a25e60bccfc658f662dfee162bb114f2ddaccb2c","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98280\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-list6zv9h4-ncgnxk\" in resourcegroup=\"ea-list-4wzbljbzdfsj\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/ea-list-4wzbljbzdfsj/providers/Microsoft.Resources/deployments/customer-infra-ea-list6zv9h4-ncgnxk/operationStatuses/08584116992369169703\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98280\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-list6zv9h4-ncgnxk\" in resourcegroup=\"ea-list-4wzbljbzdfsj\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/ea-list-4wzbljbzdfsj/providers/Microsoft.Resources/deployments/customer-infra-ea-list6zv9h4-ncgnxk/operationStatuses/08584116992369169703\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b3234b9726f3fc85edcde87feda0137418d489bd3afd0626bcb69689f1ea4472","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"a21cc835fa8cdb2ed9033f1ca18d1eb7db85fc60669d3f2f049f9f0f8e99c24d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:111]: failed to create customer resources for cluster \"cluster-candidate-4-20-kpzqjb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000793700\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-20-kpzqjb-rzndrn\" in resourcegroup=\"rg-candidate-4-20-hv7qvmmjkmk5\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-20-hv7qvmmjkmk5/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-20-kpzqjb-rzndrn/operationStatuses/08584116992454378325\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:111]: failed to create customer resources for cluster \"cluster-candidate-4-20-kpzqjb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000793700\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-20-kpzqjb-rzndrn\" in resourcegroup=\"rg-candidate-4-20-hv7qvmmjkmk5\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-20-hv7qvmmjkmk5/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-20-kpzqjb-rzndrn/operationStatuses/08584116992454378325\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c89daa424607cd7dee410ca9373d71e0c35ab6ae04c63c09dadb3673f0f3694f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"c3ed73d691c5929cdf2946a476c8b001c9536e6031f54f65e36b74ff7e76c90f","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:95]: failed to deploy customer infrastructure for back-level version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001009720\u003e: \n failed waiting for deployment \"customer-infra-cluster-ver-4-20-m6nnt5\" in resourcegroup=\"rg-cluster-back-version-gddcccf28p5t\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-back-version-gddcccf28p5t/providers/Microsoft.Resources/deployments/customer-infra-cluster-ver-4-20-m6nnt5/operationStatuses/08584116992459366081\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:95]: failed to deploy customer infrastructure for back-level version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001009720\u003e: \n failed waiting for deployment \"customer-infra-cluster-ver-4-20-m6nnt5\" in resourcegroup=\"rg-cluster-back-version-gddcccf28p5t\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-back-version-gddcccf28p5t/providers/Microsoft.Resources/deployments/customer-infra-cluster-ver-4-20-m6nnt5/operationStatuses/08584116992459366081\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"cc57aee78b3227375504cf7ced949360d61d6993c040319ea1cab2c1d1fb5e0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"2b974162c7e6d97b0de4aea836205f4f789664585153978c18ab76f65fd6ff59","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:78]: failed to create customer resources for KMS key rotation cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00091f420\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-kms-key-rotate-422-pxl46j\" in resourcegroup=\"kms-key-rotate-j9t2vs2r5nhz\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/kms-key-rotate-j9t2vs2r5nhz/providers/Microsoft.Resources/deployments/customer-infra-kms-key-rotate-422-pxl46j/operationStatuses/08584116992371388304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:78]: failed to create customer resources for KMS key rotation cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00091f420\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-kms-key-rotate-422-pxl46j\" in resourcegroup=\"kms-key-rotate-j9t2vs2r5nhz\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/kms-key-rotate-j9t2vs2r5nhz/providers/Microsoft.Resources/deployments/customer-infra-kms-key-rotate-422-pxl46j/operationStatuses/08584116992371388304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"da675a2341fc514db40cb885acb7d29eb2a6afa1d66a9f22d368a05ca6cdf8e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"1b003937044cfa4fcd5a2675364491c0f6ffb7db83c6398081a44e0f34d6bb6b","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a11f20\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-ltthln\" in resourcegroup=\"private-ingress-6mv7ddflq9wr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-6mv7ddflq9wr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-ltthln/operationStatuses/08584116992436980878\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a11f20\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-ltthln\" in resourcegroup=\"private-ingress-6mv7ddflq9wr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-6mv7ddflq9wr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-ltthln/operationStatuses/08584116992436980878\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"eaefbf5e1c83019d5b962dea3d56223c3e93de1b60d67adc79803f8228404ebb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"ab6420d788f7161106d32d50e735d0b55e0a7d0b7c8c03f40db9586c7ac5332f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-gn9sjc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000715ca0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-5-0-gn9sjc-kdgqtg\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-htm8g5gh72dw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-5-0-htm8g5gh72dw/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-5-0-gn9sjc-kdgqtg/operationStatuses/08584116992383393210\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-gn9sjc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000715ca0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-5-0-gn9sjc-kdgqtg\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-htm8g5gh72dw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-5-0-htm8g5gh72dw/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-5-0-gn9sjc-kdgqtg/operationStatuses/08584116992383393210\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f5f0967e26277b45e4b657784a4e7199419c1a1b3e1172b453c6c6ce989cbc2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"268106fe2eb57086c4eef48987e800efaac560af7778223cb42b3520a44d6749","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:110]: failed to create customer resources for cluster-subnet-reuse\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12720\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-h6k9tb\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-h6k9tb/operationStatuses/08584116992459228470\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:110]: failed to create customer resources for cluster-subnet-reuse\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12720\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-h6k9tb\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-h6k9tb/operationStatuses/08584116992459228470\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":21,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504"],"matched_occurrences":21,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101634420027953152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7026,"pr_state":"closed","pr_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","final_merged_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T11:27:46Z","started_at":"2026-09-20T11:27:46Z","completed_at":"2026-09-20T13:33:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T13:46:25.160762655Z","timing_metadata_checked_at":"2026-09-20T13:46:25.160762655Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T13:46:25.163101248Z","region_metadata_checked_at":"2026-09-20T13:46:25.163101248Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"01bf343ccafed062791df15003912369e2435fcca2f41ded1aabed6e980d7387","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101634420027953152","occurred_at":"2026-09-20T11:27:46Z","failed_at":"alert","signature_id":"5b8bff552d46579784f79a0d3b3120eddbccf7ef351d544b8715f930813682e3","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T12:53:00Z\n Ended: 2026-09-20T12:58:49Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j7953152-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j7953152-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"b71ddf1366183398b374e1610d716a27a88a350233cde6ca54a03ecea92f1717","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101634420027953152","occurred_at":"2026-09-20T11:27:46Z","failed_at":"e2e","signature_id":"b854729ec479cb89d9449d9a6d7d41e7ad976014ffc19decf951194f7d9ca05c","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a920f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-l2hrlk8rzgwm\" to finish deleting: GET https://rp.j7953152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5ce2f387-0594-454c-acf7-c24b6cbb1e1b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5ce2f387-0594-454c-acf7-c24b6cbb1e1b\",\n \"name\": \"5ce2f387-0594-454c-acf7-c24b6cbb1e1b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T12:54:10.213869417Z\",\n \"endTime\": \"2026-09-20T13:18:13.968831064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sunvib8s08n5gj9rrnedc5kn1khs4j8 still exists (deletion dispatched at 2026-09-20T12:54:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a920f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-l2hrlk8rzgwm\" to finish deleting: GET https://rp.j7953152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5ce2f387-0594-454c-acf7-c24b6cbb1e1b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5ce2f387-0594-454c-acf7-c24b6cbb1e1b\",\n \"name\": \"5ce2f387-0594-454c-acf7-c24b6cbb1e1b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T12:54:10.213869417Z\",\n \"endTime\": \"2026-09-20T13:18:13.968831064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sunvib8s08n5gj9rrnedc5kn1khs4j8 still exists (deletion dispatched at 2026-09-20T12:54:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6941/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610749217476608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6941,"pr_state":"closed","pr_sha":"5196c129c2fe0fb3f9bfc329e0463402243206bc","final_merged_sha":"5196c129c2fe0fb3f9bfc329e0463402243206bc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T09:53:43Z","started_at":"2026-09-20T09:53:43Z","completed_at":"2026-09-20T12:00:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T12:11:30.615411636Z","timing_metadata_checked_at":"2026-09-20T12:11:30.615411636Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T12:11:30.618077928Z","region_metadata_checked_at":"2026-09-20T12:11:30.618077928Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T09:53:04Z","started_at":"2026-09-20T09:53:04Z","completed_at":"2026-09-20T11:57:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T12:00:33.598053486Z","timing_metadata_checked_at":"2026-09-20T12:00:33.598053486Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T12:00:33.600936977Z","region_metadata_checked_at":"2026-09-20T12:00:33.600936977Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"161289058ece2082e780c6021f6440e9c0b15d26d604d60ca9a1e195978f2214","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","failed_at":"e2e","signature_id":"8a93b4c61b0436ed275798352e60b97494e315997f9735dcbbf3fcea1cb8b1bc","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8d220\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-cc6rq5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-cc6rq5\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-pullsecret-hcp-cluster-cc6rq5' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8d220\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-cc6rq5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-cc6rq5\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-pullsecret-hcp-cluster-cc6rq5' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}},{"row_id":"a941b8a7843c480006f270a64d8a5b19b51b3a0e9acb730eb356c00083119535","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","failed_at":"alert","signature_id":"f6030bcc4e4cc5c82d977d09b32388275720da97127703ae213df38d33273f13","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T11:27:29Z\n Ended: 2026-09-20T11:33:32Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j2086400-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2086400-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"d85c78bee8d1d6813548e19fee771f2b404a0a4ce7351aa6ecf896ef948a1227","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","failed_at":"e2e","signature_id":"39cf09c4d4c3e066082fd6d0301325efef03f8489b16e7088e47c2ad9b88bbaa","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\nUnexpected error:\n \u003c*errors.StatusError | 0xc000ac2640\u003e: \n Operation cannot be fulfilled on nodes \"k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\nUnexpected error:\n \u003c*errors.StatusError | 0xc000ac2640\u003e: \n Operation cannot be fulfilled on nodes \"k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred"}},{"row_id":"e6cc162045a2e1ad5f7b6f7db2a97387484a19646003916e45dbacd874c354e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","failed_at":"e2e","signature_id":"7315612725ec25e205b11f3adbd85547f42e9e5cff4ae5b4c955333bbd4ccd4f","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-wg24nd-tjd5hx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-wg24nd-tjd5hx\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-np-version-upgrade-cluster-wg24nd-tjd5hx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-wg24nd-tjd5hx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-wg24nd-tjd5hx\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-np-version-upgrade-cluster-wg24nd-tjd5hx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610157367627776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7025,"pr_state":"closed","pr_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","final_merged_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T09:51:22Z","started_at":"2026-09-20T09:51:22Z","completed_at":"2026-09-20T12:02:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T12:11:31.002193402Z","timing_metadata_checked_at":"2026-09-20T12:11:31.002193402Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T12:11:31.004756094Z","region_metadata_checked_at":"2026-09-20T12:11:31.004756094Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b03c9a884adc647e77e1b18fa191a5eeeba6ce6deeee6752458ac91e9e041b3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610157367627776","occurred_at":"2026-09-20T09:51:22Z","failed_at":"e2e","signature_id":"d3228063fdb4db7077f94a323b61ddb40a6e50e0a6a4ce81d902c823eb177dc2","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001ba80f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-j5f5ss4sbkp7\" to finish deleting: GET https://rp.j7627776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/676ab8d4-2d39-4758-b787-1a8b8a8dad3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/676ab8d4-2d39-4758-b787-1a8b8a8dad3f\",\n \"name\": \"676ab8d4-2d39-4758-b787-1a8b8a8dad3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T11:23:08.876040701Z\",\n \"endTime\": \"2026-09-20T11:47:13.388195572Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sumjjp5k6vfqppk7421kh5jdhmndmk0 still exists (deletion dispatched at 2026-09-20T11:23:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001ba80f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-j5f5ss4sbkp7\" to finish deleting: GET https://rp.j7627776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/676ab8d4-2d39-4758-b787-1a8b8a8dad3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/676ab8d4-2d39-4758-b787-1a8b8a8dad3f\",\n \"name\": \"676ab8d4-2d39-4758-b787-1a8b8a8dad3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T11:23:08.876040701Z\",\n \"endTime\": \"2026-09-20T11:47:13.388195572Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sumjjp5k6vfqppk7421kh5jdhmndmk0 still exists (deletion dispatched at 2026-09-20T11:23:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101600364640866304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7026,"pr_state":"closed","pr_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","final_merged_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T09:12:27Z","started_at":"2026-09-20T09:12:27Z","completed_at":"2026-09-20T11:21:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T11:28:32.587013286Z","timing_metadata_checked_at":"2026-09-20T11:28:32.587013286Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T11:28:32.589897477Z","region_metadata_checked_at":"2026-09-20T11:28:32.589897477Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"0f974f941334a44e14698348b15d174831988e5504e9e9d27fb043d8961369b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101600364640866304","occurred_at":"2026-09-20T09:12:27Z","failed_at":"alert","signature_id":"f3bf8d60286004f12d28b5f26936659fa6d1debd6a2443cbc6166170ce25bcb6","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T10:11:11Z\n Ended: 2026-09-20T10:45:10Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0866304-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0866304-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7023/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101588887028633600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7023,"pr_state":"closed","pr_sha":"6cae5a1171dd63e42d1171cdfc4ddd325d953fb9","final_merged_sha":"6cae5a1171dd63e42d1171cdfc4ddd325d953fb9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T08:26:50Z","started_at":"2026-09-20T08:26:50Z","completed_at":"2026-09-20T10:56:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T10:56:50.501476469Z","timing_metadata_checked_at":"2026-09-20T10:56:50.501476469Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T10:56:50.505736543Z","region_metadata_checked_at":"2026-09-20T10:56:50.505736543Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2b2b934509611a0e7e66acb3b999fe61265128d1009f064da68a6e1c2d973d61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7023/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101588887028633600","occurred_at":"2026-09-20T08:26:50Z","failed_at":"e2e","signature_id":"3a8359bef606bfcfc9ffd99f5dc07bf9b62993cb12fd5d570d46aa5683d99f29","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014a6168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-h2thhnsvkjqg\" to finish deleting: GET https://rp.j8633600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a58bfa81-14c0-418f-8038-eb180ea92a60\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a58bfa81-14c0-418f-8038-eb180ea92a60\",\n \"name\": \"a58bfa81-14c0-418f-8038-eb180ea92a60\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T10:14:25.908843484Z\",\n \"endTime\": \"2026-09-20T10:38:32.00607941Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sulilh97eig2ggpp4bpddu5dj29cgdm still exists (deletion dispatched at 2026-09-20T10:14:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014a6168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-h2thhnsvkjqg\" to finish deleting: GET https://rp.j8633600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a58bfa81-14c0-418f-8038-eb180ea92a60\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a58bfa81-14c0-418f-8038-eb180ea92a60\",\n \"name\": \"a58bfa81-14c0-418f-8038-eb180ea92a60\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T10:14:25.908843484Z\",\n \"endTime\": \"2026-09-20T10:38:32.00607941Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sulilh97eig2ggpp4bpddu5dj29cgdm still exists (deletion dispatched at 2026-09-20T10:14:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101575058240573440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7025,"pr_state":"closed","pr_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","final_merged_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T07:31:53Z","started_at":"2026-09-20T07:31:53Z","completed_at":"2026-09-20T09:24:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T09:33:14.856272513Z","timing_metadata_checked_at":"2026-09-20T09:33:14.856272513Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T09:33:14.86257549Z","region_metadata_checked_at":"2026-09-20T09:33:14.86257549Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f49a0483d2cfd6cf62acc18f050e8bfe7fd81a57d549cc1240ca7db758f041fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101575058240573440","occurred_at":"2026-09-20T07:31:53Z","failed_at":"e2e","signature_id":"73ffdde7c35ef7b4f90a0d9d0266878e677b3024d0af73c677985cd106c56138","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00280b770\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00280b770\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101554844631306240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7019,"pr_state":"closed","pr_sha":"42b2541179ea46321dd5a605f46f5fc661e80081","final_merged_sha":"42b2541179ea46321dd5a605f46f5fc661e80081","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T06:11:34Z","started_at":"2026-09-20T06:11:34Z","completed_at":"2026-09-20T07:52:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T07:58:53.058614227Z","timing_metadata_checked_at":"2026-09-20T07:58:53.058614227Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T07:58:53.062053816Z","region_metadata_checked_at":"2026-09-20T07:58:53.062053816Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101543782393581568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"closed","pr_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","final_merged_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T05:27:37Z","started_at":"2026-09-20T05:27:37Z","completed_at":"2026-09-20T07:27:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T07:37:53.435532307Z","timing_metadata_checked_at":"2026-09-20T07:37:53.435532307Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T07:37:53.438474796Z","region_metadata_checked_at":"2026-09-20T07:37:53.438474796Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101522358387871744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7019,"pr_state":"closed","pr_sha":"42b2541179ea46321dd5a605f46f5fc661e80081","final_merged_sha":"42b2541179ea46321dd5a605f46f5fc661e80081","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T04:02:00Z","started_at":"2026-09-20T04:02:00Z","completed_at":"2026-09-20T06:05:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T06:22:51.333529358Z","timing_metadata_checked_at":"2026-09-20T06:22:51.333529358Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T06:22:51.336988347Z","region_metadata_checked_at":"2026-09-20T06:22:51.336988347Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"469b589d40cc288317ac2017d05939d5957a51aa85041b4c230ae7113cc8c78c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101522358387871744","occurred_at":"2026-09-20T04:02:00Z","failed_at":"e2e","signature_id":"07b9cc47f1337561bab912a04c50994e47b493cb94f69944313b85e20e5e8827","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001f2540\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-g8jlwfczq87l\" to finish updating: GET https://rp.j7871744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001f2540\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-g8jlwfczq87l\" to finish updating: GET https://rp.j7871744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7024/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101522356693372928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7024,"pr_state":"closed","pr_sha":"7a275cf861815aa8598c8a212a1d953a6d2da491","final_merged_sha":"7a275cf861815aa8598c8a212a1d953a6d2da491","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T04:01:55Z","started_at":"2026-09-20T04:01:55Z","completed_at":"2026-09-20T06:17:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T06:33:46.603267219Z","timing_metadata_checked_at":"2026-09-20T06:33:46.603267219Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T06:33:46.606884807Z","region_metadata_checked_at":"2026-09-20T06:33:46.606884807Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T23:00:39Z","started_at":"2026-09-20T23:00:39Z","completed_at":"2026-09-21T01:00:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T01:10:18.789033323Z","timing_metadata_checked_at":"2026-09-21T01:10:18.789033323Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T01:10:18.792321213Z","region_metadata_checked_at":"2026-09-21T01:10:18.792321213Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"27b07e12f1f192bd7f78dd8cbe033eacaac8370a279552c184e0cb83b1946625","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"609d9384e69628bf55c86d248dad68fa3cfbe04261dd0ff9804138d57638cb07","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0016fa690\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gc2rhgk55bsk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0016fa690\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-gc2rhgk55bsk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2e228bb7993814820ec60c4b24c66f3406998c2cdbb2380f86f33cc0546a8b8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"0c0591f9586da1ceac9fcc46f897431c4bbba6e1aebadce62e31ef160dc87f07","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-dr4c56\" for cluster \"cluster-nightly-4-20-dr4c56\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d101a0\u003e: \n failed to create NodePool np-dr4c56: failed starting nodepool creation \"np-dr4c56\" for cluster \"cluster-nightly-4-20-dr4c56\" in resourcegroup=\"rg-nightly-4-20-xfvxqd785djj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-xfvxqd785djj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-dr4c56/nodePools/np-dr4c56\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-dr4c56\" for cluster \"cluster-nightly-4-20-dr4c56\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d101a0\u003e: \n failed to create NodePool np-dr4c56: failed starting nodepool creation \"np-dr4c56\" for cluster \"cluster-nightly-4-20-dr4c56\" in resourcegroup=\"rg-nightly-4-20-xfvxqd785djj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-xfvxqd785djj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-dr4c56/nodePools/np-dr4c56\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"343412e1c9b840e5afa5c7005cf3191117e995562605096bac1264f78b39f849","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"cbcfc6c0ea890936fe78377d8bd883d847e8c26fd22ca0042a2db994fd99152c","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0018004e0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0018004e0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"598a288915897077f1ec57871782bc82359490b8c9888c928e59659ea030beb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"d1a6fefdadb947de8c33d9842ac16f2fca17157a7f575a18d904be8040d389a6","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00146a6f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-hvbstq2qnv55/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00146a6f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-hvbstq2qnv55/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"80c6ffaf19876522191c9d8ea192efdacee0d054d88038a9a1c6077f5a4d9d3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"328adc8fb8c132a962c8f7e8e644e383234b1bc202be2521e5dea0f4d52718f5","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00286bb60\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-r4zrv9dgbv7h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00286bb60\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-r4zrv9dgbv7h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a30956a69e8998942198940968968d08f28ad271bd985f908929303282f54109","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"6196ac4300552dfdb8e74a1b49f7d8225f6e95469be8ee12fdcb6a58f8ef5c17","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001184fe0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-6cgfzw\" in resourcegroup=\"rg-np-skip-minor-6cgfzw-4bn8bljlps4c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-6cgfzw-4bn8bljlps4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-6cgfzw/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001184fe0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-6cgfzw\" in resourcegroup=\"rg-np-skip-minor-6cgfzw-4bn8bljlps4c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-6cgfzw-4bn8bljlps4c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-6cgfzw/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b5d356c2ec448e0b20a0f8ac1de96139801d86b7af45d81fcafc1e40af18ccd5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"c1035688eca499c52db3ca5bb0c7a77093840ac45b328b2854d0de497f1f82bb","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0029094d0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-gb4v8nm48j8x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0029094d0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-gb4v8nm48j8x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b94b0ce38bb036ab722e94e50d54878697c2e07d42158b3e18076a9e59d86a60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"e1e9f91cb19890a0eb0ac4baf7b91616569638fb83a8ccd96e69e04addc7141a","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e7440\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-mtt578qwjxhd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-mtt578qwjxhd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e7440\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-mtt578qwjxhd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-mtt578qwjxhd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bb2644ed34447667e554d68eaa4fa9c79c3dc92506fcf91ab5a627d77e404390","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"0230eab10a5b6baf7463f9373d53a9a0d5e973ea498e4480c157f3188984c7e7","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d12020\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7hkgvb\" in resourcegroup=\"rg-np-version-upgrade-7hkgvb-db7f9x6gvf59\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7hkgvb-db7f9x6gvf59/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7hkgvb/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d12020\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7hkgvb\" in resourcegroup=\"rg-np-version-upgrade-7hkgvb-db7f9x6gvf59\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7hkgvb-db7f9x6gvf59/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7hkgvb/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c4e032638db4177ec135bb881382c360cc732815dfc9512d4f7ae8207fa2116b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"4e8137a2ef56b5461da5789f436cda8495e7a5d9d716cbfd68569ebc575a8b1c","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001070ec0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-zrpt5w\" resourcegroup=\"rg-np-dg-minor-zrpt5w-4m5b8rktsw6x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zrpt5w-4m5b8rktsw6x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-zrpt5w/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001070ec0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-zrpt5w\" resourcegroup=\"rg-np-dg-minor-zrpt5w-4m5b8rktsw6x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-zrpt5w-4m5b8rktsw6x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-zrpt5w/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d50f0de2cac8cbe81adb29d388444d0f6d2f4551b9f2e7d4f8bb6064afbb7d8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2101808786619502592","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"17a023c6a46245432cc416ef8d62649f55ccb9fa39f11d5aea7cbfa4336d9de3","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001896cf0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-79mxdqgdx9t6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001896cf0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-79mxdqgdx9t6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2101491692631756800","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T02:00:37Z","started_at":"2026-09-20T02:00:37Z","completed_at":"2026-09-20T04:08:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T04:25:53.461635417Z","timing_metadata_checked_at":"2026-09-20T04:25:53.461635417Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T04:25:53.465817399Z","region_metadata_checked_at":"2026-09-20T04:25:53.465817399Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"162242c8638feff3a88871d460276fbe8106040f4802819caaac2ed7a91b73e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2101491692631756800","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"8381cc68ee86757495de46236ed3b123ff2a372a4b908f8548b0bdace5e822bb","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00118b8a0\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-7sn2sc\" in resourcegroup=\"rg-np-skip-minor-7sn2sc-x6d7knwq47tj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0555a946-aa1d-468d-a6ad-48364e5f992e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0555a946-aa1d-468d-a6ad-48364e5f992e\",\n \"name\": \"0555a946-aa1d-468d-a6ad-48364e5f992e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T03:23:26.91112939Z\",\n \"endTime\": \"2026-09-20T03:42:27.966002583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00118b8a0\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-7sn2sc\" in resourcegroup=\"rg-np-skip-minor-7sn2sc-x6d7knwq47tj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0555a946-aa1d-468d-a6ad-48364e5f992e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0555a946-aa1d-468d-a6ad-48364e5f992e\",\n \"name\": \"0555a946-aa1d-468d-a6ad-48364e5f992e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T03:23:26.91112939Z\",\n \"endTime\": \"2026-09-20T03:42:27.966002583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3d6109d42c01682bf7d294f84e0138ac5322343ec8f488c344b7397d9425c2c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2101491692631756800","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"7dea51084c53cfca68d054fdb2b8ed1e7adcbbca9101d85ead03c9a16e317eb4","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000ed0f48\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-75p2xk77rtzr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65cff58d-536e-4d46-b35b-58f67af72b80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65cff58d-536e-4d46-b35b-58f67af72b80\",\n \"name\": \"65cff58d-536e-4d46-b35b-58f67af72b80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T02:16:51.798119339Z\",\n \"endTime\": \"2026-09-20T02:35:59.041922136Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000ed0f48\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-75p2xk77rtzr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65cff58d-536e-4d46-b35b-58f67af72b80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65cff58d-536e-4d46-b35b-58f67af72b80\",\n \"name\": \"65cff58d-536e-4d46-b35b-58f67af72b80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T02:16:51.798119339Z\",\n \"endTime\": \"2026-09-20T02:35:59.041922136Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6db5150c92e4bd745d7c0c611c4905edda623de4211d69e7b29ba4e781e424ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2101491692631756800","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"933e30630642a92f4e79135b5aa54df21013442019274bf9c20132e6dffe4ecc","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011a240\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-h72vxbzhd2w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5017a36-2ffb-4b1f-9f5e-28e6a6204319\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5017a36-2ffb-4b1f-9f5e-28e6a6204319\",\n \"name\": \"b5017a36-2ffb-4b1f-9f5e-28e6a6204319\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T03:10:04.541118917Z\",\n \"endTime\": \"2026-09-20T03:29:07.643791064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011a240\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-h72vxbzhd2w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5017a36-2ffb-4b1f-9f5e-28e6a6204319\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5017a36-2ffb-4b1f-9f5e-28e6a6204319\",\n \"name\": \"b5017a36-2ffb-4b1f-9f5e-28e6a6204319\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T03:10:04.541118917Z\",\n \"endTime\": \"2026-09-20T03:29:07.643791064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T23:00:39Z","started_at":"2026-09-20T23:00:39Z","completed_at":"2026-09-20T23:17:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T23:35:24.397224451Z","timing_metadata_checked_at":"2026-09-20T23:35:24.397224451Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T23:35:24.400185642Z","region_metadata_checked_at":"2026-09-20T23:35:24.400185642Z"},"failed_at":["e2e"],"failed_test_count":52,"occurrences":[{"row_id":"010474fd588c09448e6e42bb2d85d9f37a11116aba1014233528c7b54e5ae34f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"bf9220ee57546bfb98ca09c23ce1bcafa0fc9b3ad8c908396fb22e88d8e4390f","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-sl6zqd\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed0240\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-sl6zqd\" in resourcegroup=\"admin-credential-lifecycle-test-wq5ttkkvjgbt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-wq5ttkkvjgbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-sl6zqd\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-sl6zqd\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed0240\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-sl6zqd\" in resourcegroup=\"admin-credential-lifecycle-test-wq5ttkkvjgbt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-wq5ttkkvjgbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-sl6zqd\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"076f4e213991c5f4a0a6d0b34fedb17f62ec6189d10eb742ce7fddf24a3387f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"f036046fe003926a26f5c7f66f4226f6e0b880c29bba5601c195ecc014eafa48","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001138120\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-79x6dgw9vgjh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-79x6dgw9vgjh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001138120\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-79x6dgw9vgjh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-79x6dgw9vgjh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"078b678817e64f7f27cca8a252d9ac8c28238e766f6673de0707293186cdf282","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"35fca88da7d43a424bd831003af76469f024f27d52d9191f8d38f4ebd3598aab","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706280\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-dlhm5kjf7lrn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-dlhm5kjf7lrn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000706280\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-dlhm5kjf7lrn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-dlhm5kjf7lrn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0e2e3b7b00ab1b6ff061238bd889d6442c36a3ae3a721a10203174854e79371b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"1b89ced61ee5be756db27ad1f65de34e90424823131553ce990f5e8bfac77fdd","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a465c0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vcwcfxfj2wxt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-vcwcfxfj2wxt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a465c0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vcwcfxfj2wxt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-vcwcfxfj2wxt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2867d91d2373d81ada4b2fdc4c1505e3115c1e4a494ab61418606d82e1c2b81f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"ed63787da4d3fd4cb31e6acfb7525e56bc4189b4c555b5dee49e05c0357ee203","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-7hgjxt\" at install version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00099ca60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-7hgjxt: failed starting cluster creation \"cp-ystream-upgrade-4-21-7hgjxt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-lzrxkld9dhxh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-lzrxkld9dhxh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-7hgjxt\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-7hgjxt\" at install version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00099ca60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-7hgjxt: failed starting cluster creation \"cp-ystream-upgrade-4-21-7hgjxt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-lzrxkld9dhxh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-lzrxkld9dhxh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-7hgjxt\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"32643b3ba1be2ebee6ccfc9b48d71b32e3c0edeb50a11b9e2f3acefdc751ec13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"0035badf1d52e79b46a89636294ab5175c1764ce57ed472580f51c38a17646d8","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa0100\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-htkcwk2h26jm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-htkcwk2h26jm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa0100\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-htkcwk2h26jm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-htkcwk2h26jm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"44d62174ee7e32d731e91b2ec6fc9f6b69590d0955a7bb703561676340f6e653","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"f54684b919b3a17e9b0d89e05f999162d7714a3aec1ee3f6660f91a081bd3ae2","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001194280\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-67p5h8f8824m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-67p5h8f8824m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001194280\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-67p5h8f8824m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-67p5h8f8824m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4a9ef8be8131cc830464d2676dc5ea1c0d78c4a9e8ae6d9f03de0fae713dc31f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"d7f52fda9e0a37e1b44a584e50dea99d5f305e1aec140de81d09c8899f6ecd59","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-vtj989245nwh/cluster-nightly-5-1-p6j2b7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017102a0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-p6j2b7: failed starting cluster creation \"cluster-nightly-5-1-p6j2b7\" in resourcegroup=\"rg-nightly-5-1-vtj989245nwh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-vtj989245nwh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-1-p6j2b7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-1-vtj989245nwh/cluster-nightly-5-1-p6j2b7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017102a0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-p6j2b7: failed starting cluster creation \"cluster-nightly-5-1-p6j2b7\" in resourcegroup=\"rg-nightly-5-1-vtj989245nwh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-vtj989245nwh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-1-p6j2b7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4f46f93af2a3deca4bbef53aade49639f6fbf7b4b32b81837e52b9487c808e78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"737be13fabbdd7d8938f70a23dfc8dc615ab58be6134cddbe08c7cef4614840d","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011da560\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-jxtwrm2xbmsh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-jxtwrm2xbmsh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011da560\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-jxtwrm2xbmsh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-jxtwrm2xbmsh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"56c31ee16b1738d783b9cbbb656a57317f7dc3a7cd328c0b7b9e302049596e2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"0726e178dba9603bcc13b5275955ebffef79669ebe41f4359f6650518623f84c","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009903c0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-h5k2wts9ndxl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-h5k2wts9ndxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009903c0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-h5k2wts9ndxl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-h5k2wts9ndxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5b2e350e46ea7111aa8e10f5430c514c9c5bee277ace5b0588ea900390c9b32c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"4b692397a309ba5941c3b37f467df2509c51aa2665cd1ded857f0c6b44b0037c","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-hsbkgd\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efe5c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-hsbkgd: failed starting cluster creation \"cp-ystream-upgrade-4-23-hsbkgd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ng74hd8c7cxw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-ng74hd8c7cxw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-hsbkgd\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-hsbkgd\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efe5c0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-hsbkgd: failed starting cluster creation \"cp-ystream-upgrade-4-23-hsbkgd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-ng74hd8c7cxw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-ng74hd8c7cxw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-hsbkgd\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5ddf91ee6bff947c14a76a8263d3fed564df460fe2bb8a24d12f3d4e6be77b2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"1cd687ef7bbbb14f8d590c7438121eeea9ec4020c01b7de0e34917b9f0992ab8","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00056e660\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-fr9mcq429nkx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-fr9mcq429nkx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00056e660\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-fr9mcq429nkx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-fr9mcq429nkx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"61dd80a509de8009ec9ad20433cad03bf0bfdc0d415c220d22c1cbcc40cf8843","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"207cda487ed44811f5480f8f37446d9601422530a9a36879efe9f029372d698c","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001588f60\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001588f60\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6494b477e7699ff79843e03fa0936d2e18ee76b760a27ff46407a2b438c98006","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"9934fed1fdd8af1db46791b1dfefd62cd08c923399e036e187caa7b300776d49","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-20-ptcmmmt5clqs/cluster-nightly-4-20-gng77x should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e48220\u003e: \n failed to create HCP cluster cluster-nightly-4-20-gng77x: failed starting cluster creation \"cluster-nightly-4-20-gng77x\" in resourcegroup=\"rg-nightly-4-20-ptcmmmt5clqs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-ptcmmmt5clqs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-gng77x\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-20-ptcmmmt5clqs/cluster-nightly-4-20-gng77x should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e48220\u003e: \n failed to create HCP cluster cluster-nightly-4-20-gng77x: failed starting cluster creation \"cluster-nightly-4-20-gng77x\" in resourcegroup=\"rg-nightly-4-20-ptcmmmt5clqs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-ptcmmmt5clqs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-gng77x\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"673e2a226207305a892eb3b90ff091864dc0774127e0e9fa488cbf408ebb3a4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"910729e19a1a95fb2ecb26f059eb401491443d77afb01693df52c373ec5864aa","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ee4300\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-ccgw5v5rwcjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ee4300\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-ccgw5v5rwcjt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6b7027229bcf2e1a6387bf64501a5e2157b8a9a85c2f4898a628f4e9f918d92b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"ec1323a0e04d40f520c159983f60e8cb338c88ccf1ebf099cb4705b2023d96fc","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-nmd5jz\" at install version 4.21.0-0.nightly-multi-2026-09-17-004700\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e385e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-nmd5jz: failed starting cluster creation \"cp-ystream-upgrade-4-22-nmd5jz\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-vqqsp2lxgb87\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-vqqsp2lxgb87/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-nmd5jz\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-nmd5jz\" at install version 4.21.0-0.nightly-multi-2026-09-17-004700\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e385e0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-nmd5jz: failed starting cluster creation \"cp-ystream-upgrade-4-22-nmd5jz\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-vqqsp2lxgb87\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-vqqsp2lxgb87/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-nmd5jz\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"705069724709ce8bdba875775c0b17ab372c8d9d3dd543c84fc1cc02d8f70cff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"44f5ce1f36b75dc92b37eb2219e5536cbf8dae57ad133af027cb383345b0f81f","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e260\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-v7hck72bbqhd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-v7hck72bbqhd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8e260\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-v7hck72bbqhd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-v7hck72bbqhd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"77de0b788d151c5c5928d02a43c2a1c9ceba654557af72e2aa0d1a74142930dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"51cdc90b4d51d3970f1db21a4d2ae21e451183c4709e4ff89759bb06ce3c0d56","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000962160\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-wwj9rcvvv7tw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wwj9rcvvv7tw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000962160\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-wwj9rcvvv7tw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-wwj9rcvvv7tw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7881fc30dbb3514c9f65ee2504234f43e4dd23ac6fb4fc6fc200dc4d12a9d241","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"2262ff705c982dcfb0f913112870bd6e548d66fa181c0b45a3eaaaa1d46212fa","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea4340\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-sphjhhgmnrcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-sphjhhgmnrcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea4340\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-sphjhhgmnrcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-sphjhhgmnrcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7f843b02d7b0c44ae1061b838d559961fcc7c0b3570f2c9165bcbaf336df7667","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"3deeda3dacfcc6e9613866522eff1df6ffa855f675bfcd2b3400d1a8bb0043e7","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008dc760\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9gtb4f4wnslh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9gtb4f4wnslh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008dc760\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-9gtb4f4wnslh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-9gtb4f4wnslh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"80b50cfa9af36e702c6e1968a6a7f62b45f2880b088e858b44dbfa9d512495f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"9703f084cb7b88b8745c68c7057a0dee5319018ab6983f426787450090e42982","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00055e620\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ljdf7mggpjq9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ljdf7mggpjq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00055e620\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ljdf7mggpjq9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ljdf7mggpjq9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"85d9db04927cdc17ea8a098d41b7ec1796b8d83c7bb164878ab384f05bcfeb7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"fc16bebb53c8d6af1459f162bc0d57eea7307426a6d7301cf07495177dd5b525","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010981c0\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-kw7nr95m458x\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kw7nr95m458x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010981c0\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-kw7nr95m458x\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-kw7nr95m458x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"85dc6ac31dcbdefc8a0a4d1aceed50297f3f1a7832f66e8a297fa4d82fb3bdda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"e6b78af356090863123d3e08d8801c8ba0ac603f871ed2cb326d9ca914c11f0d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-0-zlrwcn5zcl9b/cluster-nightly-5-0-fznsx6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8e1c0\u003e: \n failed to create HCP cluster cluster-nightly-5-0-fznsx6: failed starting cluster creation \"cluster-nightly-5-0-fznsx6\" in resourcegroup=\"rg-nightly-5-0-zlrwcn5zcl9b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-zlrwcn5zcl9b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-0-fznsx6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-5-0-zlrwcn5zcl9b/cluster-nightly-5-0-fznsx6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8e1c0\u003e: \n failed to create HCP cluster cluster-nightly-5-0-fznsx6: failed starting cluster creation \"cluster-nightly-5-0-fznsx6\" in resourcegroup=\"rg-nightly-5-0-zlrwcn5zcl9b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-zlrwcn5zcl9b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-5-0-fznsx6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8704e138095432862a4743d84146083e940fa4f5d8fd9a8f3c8c4559b1af738a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"5c49ce9d1f1cb82ca579db2f9e60af5804350931bbde5c891be88d49b03e87af","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-22-lxgpbcvlkvjq/cluster-nightly-4-22-55pvlk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017003e0\u003e: \n failed to create HCP cluster cluster-nightly-4-22-55pvlk: failed starting cluster creation \"cluster-nightly-4-22-55pvlk\" in resourcegroup=\"rg-nightly-4-22-lxgpbcvlkvjq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lxgpbcvlkvjq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-22-55pvlk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-22-lxgpbcvlkvjq/cluster-nightly-4-22-55pvlk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017003e0\u003e: \n failed to create HCP cluster cluster-nightly-4-22-55pvlk: failed starting cluster creation \"cluster-nightly-4-22-55pvlk\" in resourcegroup=\"rg-nightly-4-22-lxgpbcvlkvjq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lxgpbcvlkvjq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-22-55pvlk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"870642615375904b676a5a65d54ca7d9b2332fdbdac619b27ced5f39a3931470","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"8e693b389bf5da5107e36661d6ad4dccb8934c130010b5e39c954a1624c42673","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba9120\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-ln4w2tcm2dbm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-ln4w2tcm2dbm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ba9120\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-ln4w2tcm2dbm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-ln4w2tcm2dbm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"87fbba0e646fed78eaa4dac79a5e724c5f925888c351b3a64f6b0edd62119da1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"4de4857ae4514f966ab0b2b4d8690d29ba2156a7280cfb00cab15b74f8ab103e","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015d2260\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vpvbpt7ntnb7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-vpvbpt7ntnb7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015d2260\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vpvbpt7ntnb7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-vpvbpt7ntnb7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8ff996097addedd554bf81142bc81be0d1346b71a01b5e2b745887c8674efb47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"b1bb5742be13056dd2e18f31ca353f24f129705448e865edc5ca8d30ce5acf58","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-74qm4n\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004fc900\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-74qm4n: failed starting cluster creation \"cp-ystream-upgrade-5-0-74qm4n\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-k4v7sx94tvpk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-k4v7sx94tvpk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-74qm4n\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-74qm4n\" at install version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004fc900\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-74qm4n: failed starting cluster creation \"cp-ystream-upgrade-5-0-74qm4n\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-k4v7sx94tvpk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-k4v7sx94tvpk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-74qm4n\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"917888946ad0239762d035e5a1955d898cd9c9ce462cf3cdf0e7331d709997b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"b5e7a08dd6c3cbae26446e50c6c7706b0f70b5b0efc3ca7ffd303717bf777b08","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00082a8a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-nvt6rxxn6hqx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-nvt6rxxn6hqx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00082a8a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-nvt6rxxn6hqx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-nvt6rxxn6hqx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"91b8fa825b078e3fc4c91b1df0db7f32f1cdf2e5e2ca6f4d59649961cf953068","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"cc29f32f78f823cb461df160d8a795ab0f7b89379767e103cd258870d5376e6b","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000830a00\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-6fnfqsfdmsd8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-6fnfqsfdmsd8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000830a00\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-6fnfqsfdmsd8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-6fnfqsfdmsd8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"92e4191c4f5274dd1c99c816153d565ed0245646f3283446b042d87d0de9097e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"ca0e44d54763e94dfd93a8e5d50e13a45230df631e95060b93c9af16138398d7","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001712280\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-hsdttf964hqs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-hsdttf964hqs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001712280\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-hsdttf964hqs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-hsdttf964hqs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9376290b7a76ce4b69466a395a0006f8a8d038de0685b87ca5b0658b58cccee8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"8398a4952142c55a945498132d985a4eda864ceb95312242d7edc8241b8c3c26","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004d45e0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-h2dhw86blbjb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-h2dhw86blbjb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004d45e0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-h2dhw86blbjb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-h2dhw86blbjb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"940e6c953f52abdfbc9ee206e5ad32a413b87db2280b368a2977cf37382dde3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"90a8e110e2d0e56022780623aca6ac1efad98929e39705d71e89be66fdd48f3f","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001b0600\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-n4kz6s555d6f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-n4kz6s555d6f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001b0600\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-n4kz6s555d6f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-n4kz6s555d6f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"955db6193f7c4196b9d5e0ed2a7b21b82273369a5e4256dcfc45ad28d2ad358b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"2121c5292a8ded95a05fe5474e5fc5ef3ac3f89d37ac3669aef473ed2c3b43da","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-21-dhbvmhrcqxmb/cluster-nightly-4-21-vq2qnf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e966c0\u003e: \n failed to create HCP cluster cluster-nightly-4-21-vq2qnf: failed starting cluster creation \"cluster-nightly-4-21-vq2qnf\" in resourcegroup=\"rg-nightly-4-21-dhbvmhrcqxmb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-dhbvmhrcqxmb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-21-vq2qnf\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-21-dhbvmhrcqxmb/cluster-nightly-4-21-vq2qnf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e966c0\u003e: \n failed to create HCP cluster cluster-nightly-4-21-vq2qnf: failed starting cluster creation \"cluster-nightly-4-21-vq2qnf\" in resourcegroup=\"rg-nightly-4-21-dhbvmhrcqxmb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-dhbvmhrcqxmb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-21-vq2qnf\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"998dbccd39c4e6c3ae73f1544237e008a4afc11331317952a30352b4fb3c43e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"d071dca948f099738552907e3ac446399822a96c75018da19fa4fb514960b791","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd0b00\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-czglrvgjd4wd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-czglrvgjd4wd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd0b00\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-czglrvgjd4wd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-czglrvgjd4wd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a12d8bc4791c183459f38a2edb5062a6a9b6044bec37e238d41bdf63612ce1bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"3fcbeffbfcb7999ced44c9ba14ff662338696cdba1dd77ab618e21b94c73c35e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b96280\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-8vkz7sh68ztg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-8vkz7sh68ztg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b96280\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-8vkz7sh68ztg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-8vkz7sh68ztg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a1bfcd6c993afab424a439d9f8ecca510166a09fbe1c5f90fa2057c5bf3147b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"446e04f4f7e6b9c7f65a0613b14639d4c40ffdf6cb08a3fc64d946bb72b161fd","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000703300\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4v8qfcq5rkm6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-4v8qfcq5rkm6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000703300\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4v8qfcq5rkm6\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-4v8qfcq5rkm6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a736e69c9f38d642543622703ed67c0f923b33a0e637c30e694054194433c9cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"cb9f12b0fb85348fc974d4c7f92ba92c9dc3e09d94ab423572856292667ab5ed","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae840\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-gvdhwqzjpkvw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-gvdhwqzjpkvw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fae840\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-gvdhwqzjpkvw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-gvdhwqzjpkvw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a7af49e7792725dff0f0cb2a9b6dece613d7472ebe16e1f19f24babc93875f25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"330777c600b2c414fb1e06afc890222d3aa747a1bfc6c96b43ff07b15fd45d2b","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-4ht8kz\" at install version 5.0.0-0.nightly-multi-2026-09-16-151028\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00091a180\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-4ht8kz: failed starting cluster creation \"cp-ystream-upgrade-5-1-4ht8kz\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-8g5fhjqmkkfm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-8g5fhjqmkkfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-4ht8kz\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-4ht8kz\" at install version 5.0.0-0.nightly-multi-2026-09-16-151028\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00091a180\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-4ht8kz: failed starting cluster creation \"cp-ystream-upgrade-5-1-4ht8kz\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-8g5fhjqmkkfm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-8g5fhjqmkkfm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-4ht8kz\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"aea1b0e2046e6e8b6789cce916bdfee84bc47e9ecefde7be15a984577632b156","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"8b287cd672b9ce5c760ea23a53dddea3b4f9f998a16fc2807bcd881b62d12fa8","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f802c0\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-gtngpzwwlv6t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-gtngpzwwlv6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f802c0\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-gtngpzwwlv6t\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-gtngpzwwlv6t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b2122621498ecdc6e00b69e468829106aa2d4e8f88a9e0b2210fce6a2a03a35f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"0ce1d88e3db733abc7156b9265acbfe26718a282fc729d00d437842a310dcd8a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da4260\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-bg5cnjhsfbw7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bg5cnjhsfbw7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000da4260\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-bg5cnjhsfbw7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-bg5cnjhsfbw7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"bcf5a611264a9c8ac2ed7d90e348577419841bb92a4036306f869c71fc762c5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"f4595528f2155d256fb2bac014a8f2cdd20e8e748731ae1d9a50be16b84426f9","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005f65e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-rv5l5kjnz7zn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-rv5l5kjnz7zn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005f65e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-rv5l5kjnz7zn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-rv5l5kjnz7zn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c62224d151d435fe7092384b3d45ede538089db54ade5c49285005d02a0cc57d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"a40e7b1a5607f31536cb5cd9f68551075682d33d966e6099cae407be5eb884db","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001090180\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zbxr5pwmq2ph\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-zbxr5pwmq2ph/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001090180\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-zbxr5pwmq2ph\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-zbxr5pwmq2ph/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c877ce9e65d1ad409aea869bb046956a1fac05a332aa08cac902d6f945170866","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"21d22b8dae517e3e2b72ee69216b49b53e62a80fc88bd50de4149218570a2168","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-8z55k6 with version 4.21.0-0.nightly-multi-2026-09-17-004700\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019eb40\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-8z55k6: failed starting cluster creation \"np-version-upgrade-cluster-8z55k6\" in resourcegroup=\"rg-np-version-upgrade-8z55k6-dnq6cgn6rt7w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-8z55k6-dnq6cgn6rt7w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-8z55k6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-8z55k6 with version 4.21.0-0.nightly-multi-2026-09-17-004700\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019eb40\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-8z55k6: failed starting cluster creation \"np-version-upgrade-cluster-8z55k6\" in resourcegroup=\"rg-np-version-upgrade-8z55k6-dnq6cgn6rt7w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-8z55k6-dnq6cgn6rt7w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-8z55k6\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"cf8f81037b7e19ccd36610805fd16ae47bbe9046804bbca9d0bbc2cb67f9ec01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"54559b83068e8ae8910f708b852aa10a1bfe8f38a82051dac25e3b3c2c041dbc","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce23e0\u003e: \n failed to create HCP cluster ea-listjbg4jg: failed starting cluster creation \"ea-listjbg4jg\" in resourcegroup=\"ea-list-tjtnqprlfp76\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-tjtnqprlfp76/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listjbg4jg\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce23e0\u003e: \n failed to create HCP cluster ea-listjbg4jg: failed starting cluster creation \"ea-listjbg4jg\" in resourcegroup=\"ea-list-tjtnqprlfp76\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-tjtnqprlfp76/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listjbg4jg\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d2779f977677d8c5d7decea2af64bb60210a6ff4def37466cae61712a3a41018","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"051ffac544e878c8e8e19b0de538556612bc6ea491ec662a5d5e352d905c7b58","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111e180\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-svx7g8lfp7vc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-svx7g8lfp7vc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111e180\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-svx7g8lfp7vc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-svx7g8lfp7vc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d42b37ad7f3e6deec9ff4f2ccb4fbc2c37e8427d9b778b02ad8c02429889df9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"d97958181598a8ed71cf272755962a1a133abe4c9b76728f8a3433630aa69eeb","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001620280\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-26dbbmz7n4mr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-26dbbmz7n4mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001620280\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-26dbbmz7n4mr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-26dbbmz7n4mr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d6cfe226a5876450d59e4ec8f1d1fe8f30a23c4e5b17608a1eddf9ab685f7f65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"463acd71cbfe6a54942c3369058385b4a3790957af355b5aee86ed77a281a952","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ac300\u003e: \n failed to create HCP cluster np-dg-minor-ljtcrq: failed starting cluster creation \"np-dg-minor-ljtcrq\" in resourcegroup=\"rg-np-dg-minor-ljtcrq-lg89m2gfjzvw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-ljtcrq-lg89m2gfjzvw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-ljtcrq\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ac300\u003e: \n failed to create HCP cluster np-dg-minor-ljtcrq: failed starting cluster creation \"np-dg-minor-ljtcrq\" in resourcegroup=\"rg-np-dg-minor-ljtcrq-lg89m2gfjzvw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-ljtcrq-lg89m2gfjzvw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-ljtcrq\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d8e23fae29b905ef7b3834962c7f108e25e22850f015fb9ecc53af51320c595e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"e1f2463dbdd365ac2c40122a7bd57873d9d3da823de699c668c0f0b4da3bf256","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c80100\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xhw5c5qsqn8w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xhw5c5qsqn8w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c80100\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xhw5c5qsqn8w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-xhw5c5qsqn8w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e8dbf7b44e14a5b116e01723e5c026f9d81c2b898e563abacb991fad0c0d6090","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"53503846cd73044f90ce87f6819d59a29a0c5aeb170996d62d78a12b6c18d33f","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-ltgr76 at version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e004a0\u003e: \n failed to create HCP cluster np-skip-minor-ltgr76: failed starting cluster creation \"np-skip-minor-ltgr76\" in resourcegroup=\"rg-np-skip-minor-ltgr76-pdv7k5rzckpw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-ltgr76-pdv7k5rzckpw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-ltgr76\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-ltgr76 at version 4.22.0-0.nightly-multi-2026-09-17-071442\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e004a0\u003e: \n failed to create HCP cluster np-skip-minor-ltgr76: failed starting cluster creation \"np-skip-minor-ltgr76\" in resourcegroup=\"rg-np-skip-minor-ltgr76-pdv7k5rzckpw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-ltgr76-pdv7k5rzckpw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-ltgr76\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ea92f89cdc6c8ddff7a5ba2fad6d45292bbae724a7fb50bd30946626637274c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"be9743a3b7525b79eee4c11217ab516d2317647c36b86a981d9439f483ce73e4","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00170c320\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-49vc866tqlcg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-49vc866tqlcg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00170c320\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-49vc866tqlcg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-49vc866tqlcg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f15dbbfd62a98a87fafc88b7cf492bf3f56f57743d9186185fc91248341c16db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"b3bf8aee173cbf7a075ad4f0093992fdf2f9d3fdbdd4c79ed228a07fc41be6a7","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00125e6e0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-jjhlbwqk4n2l\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-jjhlbwqk4n2l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00125e6e0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-jjhlbwqk4n2l\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-jjhlbwqk4n2l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ff0b4b7e937c484a00259caa0c8a7fc646e04066f6f5ff1f47a97b5ebefcfa0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2101808786648862720","occurred_at":"2026-09-20T23:00:39Z","failed_at":"e2e","signature_id":"431ac06c4759e7d928f509a50e2760ae01dbe5db17da9c2a8bfbf5a018a1755a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-23-74m2d8rqxrmn/cluster-nightly-4-23-whfqfk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005102a0\u003e: \n failed to create HCP cluster cluster-nightly-4-23-whfqfk: failed starting cluster creation \"cluster-nightly-4-23-whfqfk\" in resourcegroup=\"rg-nightly-4-23-74m2d8rqxrmn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-74m2d8rqxrmn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-whfqfk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-nightly-4-23-74m2d8rqxrmn/cluster-nightly-4-23-whfqfk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005102a0\u003e: \n failed to create HCP cluster cluster-nightly-4-23-whfqfk: failed starting cluster creation \"cluster-nightly-4-23-whfqfk\" in resourcegroup=\"rg-nightly-4-23-74m2d8rqxrmn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-74m2d8rqxrmn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-whfqfk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":52,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":52,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T02:00:37Z","started_at":"2026-09-20T02:00:37Z","completed_at":"2026-09-20T02:16:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T02:32:02.46826847Z","timing_metadata_checked_at":"2026-09-20T02:32:02.46826847Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T02:32:02.47161766Z","region_metadata_checked_at":"2026-09-20T02:32:02.47161766Z"},"failed_at":["e2e"],"failed_test_count":56,"occurrences":[{"row_id":"002ce18f2c1398e91553d473e662731000501e161ef3944cb9e50116eab2014c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"22f4a53d1fb05825ca9b8ce928e01759b6a5fe1374579d62f59affa851a7cf43","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-s7r4c7 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106a240\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-s7r4c7: failed starting cluster creation \"np-version-upgrade-cluster-s7r4c7\" in resourcegroup=\"rg-np-version-upgrade-s7r4c7-j6xlvp2tvngg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-s7r4c7-j6xlvp2tvngg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-s7r4c7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-s7r4c7 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106a240\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-s7r4c7: failed starting cluster creation \"np-version-upgrade-cluster-s7r4c7\" in resourcegroup=\"rg-np-version-upgrade-s7r4c7-j6xlvp2tvngg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-s7r4c7-j6xlvp2tvngg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-s7r4c7\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"060fad763b79cf650d7f66c5b9474ca912d377a44fc230365869dbc6a8dec85d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"588cf8f8147dbe85dbbb259949b40126c92612806398b5b6d536d46ddea65806","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a32720\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-txmr84mqn85m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-txmr84mqn85m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a32720\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed starting cluster creation \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-txmr84mqn85m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-txmr84mqn85m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"0c125da09e8865272a5f7ecaf0a7c9b3230cd33cf44de259383ad5696a013ecf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"2009e10126e3e1bd18661ed5139d7b2050395ed3a6f9c56bdb146b87a1468837","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00098eb60\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-qmqkl4kkn5bh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qmqkl4kkn5bh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00098eb60\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-qmqkl4kkn5bh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qmqkl4kkn5bh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"11fd2975fca6c136a234b97a3c9d7f1ccba2cabec2ea9a7c671547203f467b2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"b5ba034a7e7faa6513a8362334aa2349a38e1f15cea70f857cb71a81269e4916","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vq25fp with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013820e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-vq25fp: failed starting cluster creation \"np-version-upgrade-cluster-vq25fp\" in resourcegroup=\"rg-np-version-upgrade-vq25fp-ppblgs9mvcst\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-vq25fp-ppblgs9mvcst/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-vq25fp\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vq25fp with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013820e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-vq25fp: failed starting cluster creation \"np-version-upgrade-cluster-vq25fp\" in resourcegroup=\"rg-np-version-upgrade-vq25fp-ppblgs9mvcst\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-vq25fp-ppblgs9mvcst/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-vq25fp\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"14cdcb6048899c219c4735259868a4d134989cd7ed8e498da82a57058a851426","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"8eef0192afaf496aef37e99995ae4991f9995de462839a3bc60443aa47866b4d","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d142a0\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-5pgjhrzg2pjb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5pgjhrzg2pjb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d142a0\u003e: \n failed starting cluster creation \"private-kv-cluster\" in resourcegroup=\"private-keyvault-5pgjhrzg2pjb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5pgjhrzg2pjb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"20cb8536ab0980a3898af318b4d046d59d4a8f8e79a792a60524395ff63b4fea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"e1b91c0003dfe8287f87363a6d3af22ea60586676659b4a066752822f9c4c009","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010d8660\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-x5qxnlrhg4qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010d8660\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-x5qxnlrhg4qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"24e1fbf8013e1b4f111992724e0ecd9620d6f7e6900240b1bd91a9f9a65679d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"1daa02e05dbdc0b2d1cf1d494fe0760919fe977375ff26fe7b7ceb2569b3ff41","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98100\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-t9tfksbtnnnj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-t9tfksbtnnnj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98100\u003e: \n failed starting cluster creation \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-t9tfksbtnnnj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-t9tfksbtnnnj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-20\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2a44ad8fbde256743ecce46244f0a17efd2ef69507dc52b10183817afa58e372","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"f35c0eb3414354887d98cdba9e0ece3c1172c182dfac94f2753434d6783656d0","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100e300\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-8kv8zbz2tm8r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8kv8zbz2tm8r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100e300\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed starting cluster creation \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-8kv8zbz2tm8r\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-8kv8zbz2tm8r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"2f70ed13178cbee683fff99023fe997a06f8d29cc8ebc89c56d8de3563e874a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"22aefaad04ec58d0bb6b547b1a1b3ad95ec0618155b7a855fb859bb3539afdcf","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b9e2a0\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-n6chlnjrnfw7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-n6chlnjrnfw7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b9e2a0\u003e: \n failed to create HCP cluster ea-cluster: failed starting cluster creation \"ea-cluster\" in resourcegroup=\"external-auth-cluster-n6chlnjrnfw7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-n6chlnjrnfw7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"349b1c424f96ec82c27fd2f1923046361886faf22fec22c146840b10a299777a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"bf612fbca19d7ba02181eea2697013613e523a9feaeec28f6e977ea886ebd7cd","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009d68c0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wrrfclhbrfzv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-wrrfclhbrfzv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009d68c0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed starting cluster creation \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wrrfclhbrfzv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-wrrfclhbrfzv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3b4888a3bb6432196ac686e6117612442ae57a3b89c79fb8340f8e208fa50e48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"0fe0f255a1fd9565d5b7035ac907cfe87a46d541f77f6bab233e15a30fe08e4e","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009f25c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-x9z6mpmx68pg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-x9z6mpmx68pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009f25c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed starting cluster creation \"node-limit-cluster\" in resourcegroup=\"node-limit-test-x9z6mpmx68pg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-x9z6mpmx68pg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"3be11a632be9a40fe915be681fcc864f8e23c0de90e2065a4d765517c9b20781","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"bf3ea225188ed6540e28e48f145437b61e1711f884c853ee13d450683d09163b","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00051c380\u003e: \n failed to create HCP cluster np-downgrade-wqfpxg: failed starting cluster creation \"np-downgrade-wqfpxg\" in resourcegroup=\"rg-np-downgrade-wqfpxg-9jb4tz7nhbwf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-wqfpxg-9jb4tz7nhbwf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-wqfpxg\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:580]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00051c380\u003e: \n failed to create HCP cluster np-downgrade-wqfpxg: failed starting cluster creation \"np-downgrade-wqfpxg\" in resourcegroup=\"rg-np-downgrade-wqfpxg-9jb4tz7nhbwf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-wqfpxg-9jb4tz7nhbwf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-wqfpxg\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"429b98234a69ee227f2bb0da14923b2ee74166eb7ab931f6a23b4357d1007287","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"abfa6c3bcad8ba9636e639bcf67ad9f2d23c2c5e09ef5ab42804ca777090585f","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097c1a0\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-rt4sdwnmhjg8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-rt4sdwnmhjg8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097c1a0\u003e: \n failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-rt4sdwnmhjg8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-rt4sdwnmhjg8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4bc7cc4077da7e42f20b295cb347b0815be8f8bccb833c766c0a88e67572f919","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"11f627ea2f83c9553f9eae34198a8c69b4a402111e6e9f67c4c5ae2ec2a9f2a2","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e2ef00\u003e: \n failed to create HCP cluster ea-listztn2b4: failed starting cluster creation \"ea-listztn2b4\" in resourcegroup=\"ea-list-hm9dzmjv79f7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-hm9dzmjv79f7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listztn2b4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e2ef00\u003e: \n failed to create HCP cluster ea-listztn2b4: failed starting cluster creation \"ea-listztn2b4\" in resourcegroup=\"ea-list-hm9dzmjv79f7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-hm9dzmjv79f7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listztn2b4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"4c2bbab85c4237ff27ff2f40663fa5e6811b3e0ed0741d65d70c9fe734a2a573","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"c0f4bd59f8cf7d2eae33429f648303444db68d7251f4a49c971f628f3abc4da4","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00063a240\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-wvrp2pc66qwb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-wvrp2pc66qwb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00063a240\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-wvrp2pc66qwb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-wvrp2pc66qwb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5380d88bef0d71db657a7a5ca43c87078340b87e5a34e23b15629f4df5f84e98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"8fb25f4e8459de9d88c4ce1a26e2e1d2d5c47224bd27ba215a6b00e68a8ec72d","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000176300\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nscjvblfvq4j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-nscjvblfvq4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000176300\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed starting cluster creation \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nscjvblfvq4j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-nscjvblfvq4j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"598f075fc7245fc890fc644bf846c41a4c6bd13bef73dddb8ef4515601199ce2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"ff472dab77219bb199206975ffe140908aae5d47dd2d0207429aabb27c7fab40","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d90400\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-6gp2tzwpdgfj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-6gp2tzwpdgfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d90400\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed starting cluster creation \"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-6gp2tzwpdgfj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-6gp2tzwpdgfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"5fb644a15c66ae077acaa67818ee1c90c1e409d627f0058741acec3f546d8736","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"d2fb9b24ef565c017137a2c2415b0bb0571aab892399bee2bfadbea4eefa7b5a","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a507e0\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-sngn82fbq5kb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-sngn82fbq5kb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:111]: failed to begin creation of first cluster \"basic-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a507e0\u003e: \n failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"customer-rg-sngn82fbq5kb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-sngn82fbq5kb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"620f4641b25fc4ced16395f231c1643f50a8d373f80b30fc00bbc92855255489","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"041f672a3e18a2b28c596e30271d2c9aa88c28c3ee7aa0cbb42b72930798fee2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103fbe0\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pxc7rnp8r6fp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-pxc7rnp8r6fp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103fbe0\u003e: \n failed starting cluster creation \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-pxc7rnp8r6fp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-pxc7rnp8r6fp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"663d0273d2f766a93b7178460253de6b227ec1bfdeac755244cda192e3074587","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"ff171e14a300a05a23f6027ecc4877b5b8e3b0abb6ddefac1a264c34b7db81db","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00016c7e0\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xdqkfgfxb45q\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-xdqkfgfxb45q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00016c7e0\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xdqkfgfxb45q\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-xdqkfgfxb45q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6bc65ed59b5dc6364e9f8c9730e2a599ab7805cbe55768266aa03e0428e4e899","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"151bc8a39c568a5acc1d67a76acc23403f08945c62637d077e8031bb8940e682","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a865e0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-mbtf6fzsh5cg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-mbtf6fzsh5cg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a865e0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed starting cluster creation \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-mbtf6fzsh5cg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-mbtf6fzsh5cg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"6da2f4920cef61f3c83c5e2a1fa8f1e0a51ae5db6f32d147aee19e5ba53fd071","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"54f1d9a0c6cd9f8dfa1399daa2e94fc81a31129243e5518016205a837d888150","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005f83a0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-mnf2gjhkwdh8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-mnf2gjhkwdh8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005f83a0\u003e: \n failed to create HCP cluster patch-tags-cluster: failed starting cluster creation \"patch-tags-cluster\" in resourcegroup=\"patch-tags-mnf2gjhkwdh8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-mnf2gjhkwdh8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7036a44f51438f56f1d8a11799abdfffe11337243932e6650aa7fcf69bd49c89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"a2b30b04fab1e84ce5ed32dc490c3554a10695067ca8a8072c99db38f1901b87","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-5lmjgr\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c20e60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-5lmjgr: failed starting cluster creation \"cp-ystream-upgrade-4-21-5lmjgr\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-slwwdk52cxx8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-slwwdk52cxx8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-5lmjgr\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-5lmjgr\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c20e60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-5lmjgr: failed starting cluster creation \"cp-ystream-upgrade-4-21-5lmjgr\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-slwwdk52cxx8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-slwwdk52cxx8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-5lmjgr\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"732b8c9f8e3809c901656db99d92280ae9564b8e138eee240e5e7395055b740c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"ff61025809bf6ec91ec3f088fbb166e1c4ec275b6978c0cc6520845e651c40ec","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-2twcsk\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d961a0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-2twcsk: failed starting cluster creation \"cluster-zstream-5-0-2twcsk\" in resourcegroup=\"rg-zstream-upgrade-5-0-twcbv9jck949\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-twcbv9jck949/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-2twcsk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-2twcsk\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d961a0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-2twcsk: failed starting cluster creation \"cluster-zstream-5-0-2twcsk\" in resourcegroup=\"rg-zstream-upgrade-5-0-twcbv9jck949\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-twcbv9jck949/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-2twcsk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"736580832924c2d075febb9a35a23e211dea749ab33018a4b8fdb6b24c038a9b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"3edae431ff77e83dcdf6bd1c2eb663743a93ce3c9b7cb3a54be975f2dcfb6d89","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005f2220\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-bg2t6zkc4bgb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-bg2t6zkc4bgb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005f2220\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed starting cluster creation \"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-bg2t6zkc4bgb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-bg2t6zkc4bgb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"73aa2c8ddd051fe5125a6c92ac4d2010be3737a8222a96a994ccb78ecdabb7a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"740e4b12b18f7ec3abcb59e913c3aac96c3fe7b74097b2e379548b0dd52e587c","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004c6b80\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-mn55wgr9jx2b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-mn55wgr9jx2b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004c6b80\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-mn55wgr9jx2b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-mn55wgr9jx2b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"76cbdd3e71f67844cca83e03aecb0f02eb80c8180dfddd5b0ac2f8bd413f83ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"f828621d2f402f351208ccd28b76304213210773de0719bb3782f81d89c3d726","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-tlrzcf\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d53580\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-tlrzcf: failed starting cluster creation \"cp-ystream-upgrade-5-0-tlrzcf\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-vxzlm59qqbzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-vxzlm59qqbzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-tlrzcf\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-tlrzcf\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d53580\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-tlrzcf: failed starting cluster creation \"cp-ystream-upgrade-5-0-tlrzcf\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-vxzlm59qqbzm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-vxzlm59qqbzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-tlrzcf\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"7f563dd5039926e46898d0195bec039d0bd92816702d34b27eef8a6d778a9271","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"f71cf1119b72fb226f585c480f5ba078edf2dbda547dcc690f2857e234bc3df7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-n5f2kcpc4z48/cluster-candidate-5-0-kvwt97 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d12440\u003e: \n failed to create HCP cluster cluster-candidate-5-0-kvwt97: failed starting cluster creation \"cluster-candidate-5-0-kvwt97\" in resourcegroup=\"rg-candidate-5-0-n5f2kcpc4z48\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-n5f2kcpc4z48/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-kvwt97\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-n5f2kcpc4z48/cluster-candidate-5-0-kvwt97 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d12440\u003e: \n failed to create HCP cluster cluster-candidate-5-0-kvwt97: failed starting cluster creation \"cluster-candidate-5-0-kvwt97\" in resourcegroup=\"rg-candidate-5-0-n5f2kcpc4z48\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-n5f2kcpc4z48/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-5-0-kvwt97\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"81eca1090eb4fc4399ee9065b0a64c415672997e18a6f22d5f754bd6b5170e32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"fbf207641738ef9568ce0804e6984e482f34e3776ee8de4a2b6d63967e228570","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0c300\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-64mcmkw4t8m7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-64mcmkw4t8m7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0c300\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-64mcmkw4t8m7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-64mcmkw4t8m7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8317cb9ff3ec73bc80611ccac6e2944ab35d6cbb940e32224bb635e6b0479103","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"a46fb253a87328078c6120731291d64f564618c7a18531510d4d69f7772b50be","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-bnbchs with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00107e220\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-bnbchs: failed starting cluster creation \"np-version-upgrade-cluster-bnbchs\" in resourcegroup=\"rg-np-version-upgrade-bnbchs-xqfkbsdkktrs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-bnbchs-xqfkbsdkktrs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-bnbchs\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-bnbchs with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00107e220\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-bnbchs: failed starting cluster creation \"np-version-upgrade-cluster-bnbchs\" in resourcegroup=\"rg-np-version-upgrade-bnbchs-xqfkbsdkktrs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-bnbchs-xqfkbsdkktrs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-bnbchs\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"851f234fee15b7a2ddfea814e1262724d409084d017d12f674e48a576efa01d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"2cd4eef6dce6c922a8e75708f23338ab8e87f52778806c126f46b6e66a6fa757","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009981a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-7k968d6mzvsn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-7k968d6mzvsn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009981a0\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed starting cluster creation \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-7k968d6mzvsn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-7k968d6mzvsn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cx-rg-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8c440a91cca351b41f0f1a563ef868bc66d0e56805face598d15b6567fb56652","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"2e3fe351111002c119c19d9dea7ccffdf52c6ecce3468c097eb0df92ee24040c","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d4a5a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-kp4x5qljgxr2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kp4x5qljgxr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d4a5a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed starting cluster creation \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-kp4x5qljgxr2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kp4x5qljgxr2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-delete-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"8e559b578e2e1fc5f00a65a4bfea60432da8ceee8086ac5fb1a84845de7cc5c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"d2f9b1d6a4dc2c4e11c24b7d6d1e51dfcc55c35723b8be324aa376ec4482623c","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000828560\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-2vb2bmn9nggf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-2vb2bmn9nggf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000828560\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed starting cluster creation \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-2vb2bmn9nggf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-2vb2bmn9nggf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"95ab1270e2753242c3d2fa2c808768ce2391a14da7ea136dcc4fea09734b7ddc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"ce692d08341314eead0cba8fe7a926c16f7053bb667d0b3113ec18c73d2c7ff5","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-gzqd2l at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001986240\u003e: \n failed to create HCP cluster np-skip-minor-gzqd2l: failed starting cluster creation \"np-skip-minor-gzqd2l\" in resourcegroup=\"rg-np-skip-minor-gzqd2l-w6r9n6665dqd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-gzqd2l-w6r9n6665dqd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-gzqd2l\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-gzqd2l at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001986240\u003e: \n failed to create HCP cluster np-skip-minor-gzqd2l: failed starting cluster creation \"np-skip-minor-gzqd2l\" in resourcegroup=\"rg-np-skip-minor-gzqd2l-w6r9n6665dqd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-gzqd2l-w6r9n6665dqd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-gzqd2l\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9aaf52123d079224ca950e49dce141585da21e44f4be590364fa67f426a26c51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"112fe54e3a4ac97241812ae495e19957375dd2aedc5ee183bdabbc212cb22b14","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f92300\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-j7crvh9fmbpf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-j7crvh9fmbpf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f92300\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-j7crvh9fmbpf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-j7crvh9fmbpf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"9c4ff1a6f633dc51edba6f05b48ad421ab4fb8915fe8cc4cbe255fe84e8436bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"8163767264e2f2a2fe4559b3d9e8fb700c29c3b53c21cdf589c6ebe8ef79d159","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-hctglvwd7lbd/cluster-candidate-4-21-pcl5b8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5a460\u003e: \n failed to create HCP cluster cluster-candidate-4-21-pcl5b8: failed starting cluster creation \"cluster-candidate-4-21-pcl5b8\" in resourcegroup=\"rg-candidate-4-21-hctglvwd7lbd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hctglvwd7lbd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-pcl5b8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-hctglvwd7lbd/cluster-candidate-4-21-pcl5b8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5a460\u003e: \n failed to create HCP cluster cluster-candidate-4-21-pcl5b8: failed starting cluster creation \"cluster-candidate-4-21-pcl5b8\" in resourcegroup=\"rg-candidate-4-21-hctglvwd7lbd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hctglvwd7lbd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-pcl5b8\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a2c61315a828dd346f1938e540372e71d194510cb3bc5d695b0c999fed8c7caa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"a34a7fa3e9d1f4ab0c95c2da0b2b4e7dffcf8ceca71238aa69ba9b23119b775e","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013ba240\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-4dwl45gd226v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-4dwl45gd226v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013ba240\u003e: \n failed to create HCP cluster missing-mi-hcp-cluster: failed starting cluster creation \"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-4dwl45gd226v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-4dwl45gd226v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/missing-mi-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"a5a0b5e3b563d26de4fe426df2de9ad9747b540c49c6cbd9fb55608f49093c44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"1b55e9418c26b0d134153b6a2d57fae30390b593c966c5993d9823c107365902","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cd0200\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-c2wrzss9lmzg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-c2wrzss9lmzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cd0200\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed starting cluster creation \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-c2wrzss9lmzg\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-c2wrzss9lmzg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b83afbd922c77e5271792a9965de64f7e6cb36bdeae7b11e64744be8cbf61438","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"b2ecf4c850ed1058543e7448874c8b4a516b8a4f9ecdf3e61ff1902b09377abb","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-4jlf9l\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011021a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-4jlf9l: failed starting cluster creation \"cp-ystream-upgrade-4-22-4jlf9l\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-7bl65fwns2ft\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-7bl65fwns2ft/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-4jlf9l\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-4jlf9l\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011021a0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-4jlf9l: failed starting cluster creation \"cp-ystream-upgrade-4-22-4jlf9l\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-7bl65fwns2ft\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-7bl65fwns2ft/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-4jlf9l\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b86cec2b693ed3ec30889397708f40f0917ee466a1b4314d55ec8e516a7b30a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"b551ad34ef00d3ca6022762cd167514b6045e84363efafdb07cb7bcc20894f44","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fee700\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-nv4b5xfbr7p4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-nv4b5xfbr7p4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fee700\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-nv4b5xfbr7p4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-nv4b5xfbr7p4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b923e7f76c2f2c6f214eaba7dd748c5dab7d1af60323c0a3f713d2670abd2fea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"951b56bf930e1710f9c89251b9a92d9c434764ebbfa79c2d4c4a43729ba1ba7d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-klztbq\" with version 4.20.37 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f36460\u003e: \n failed to create HCP cluster cluster-zstream-4-20-klztbq: failed starting cluster creation \"cluster-zstream-4-20-klztbq\" in resourcegroup=\"rg-zstream-upgrade-4-20-ngg22n4hfh9b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-ngg22n4hfh9b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-klztbq\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-klztbq\" with version 4.20.37 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f36460\u003e: \n failed to create HCP cluster cluster-zstream-4-20-klztbq: failed starting cluster creation \"cluster-zstream-4-20-klztbq\" in resourcegroup=\"rg-zstream-upgrade-4-20-ngg22n4hfh9b\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-ngg22n4hfh9b/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-klztbq\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"b985bd5e05cc2ea5eed8d460f49b55cbc8ae00de44209c78d140d4f1e8e15531","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"1e49ee1e36e7e1cd42104eac5bdf587b01098b06930d85e82c33036cb9279e20","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c2240\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-fgjslmnb248d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-fgjslmnb248d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c2240\u003e: \n failed to create HCP cluster cilium-cl: failed starting cluster creation \"cilium-cl\" in resourcegroup=\"cni-cilium-fgjslmnb248d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-fgjslmnb248d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cilium-cl\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ba3396198a57638d0d2a34411f8d3580d990763c15e88e2fd4abd507ab164caa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"91d27459306acfba0e1f7f9f09b3b58287ec53721ce8d78be8e6b545569e399f","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc6820\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-x9jkvckg2vfc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-x9jkvckg2vfc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc6820\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-x9jkvckg2vfc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-x9jkvckg2vfc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c6d8946ec217fbbfd638a9343a4e46a0a36e7546d0e701770138560878c182be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"69e9e9783ce65a18d776db022e2404a222c01c581364d8a2e36b96160faf315d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-k44cpbtv6cd7/cluster-candidate-4-20-vnlhgk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac76e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-vnlhgk: failed starting cluster creation \"cluster-candidate-4-20-vnlhgk\" in resourcegroup=\"rg-candidate-4-20-k44cpbtv6cd7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-k44cpbtv6cd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-vnlhgk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-k44cpbtv6cd7/cluster-candidate-4-20-vnlhgk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac76e0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-vnlhgk: failed starting cluster creation \"cluster-candidate-4-20-vnlhgk\" in resourcegroup=\"rg-candidate-4-20-k44cpbtv6cd7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-k44cpbtv6cd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-20-vnlhgk\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"c7599e4bf7f8a5532f1e6fe4aa0cb6ec0d7ec330489e821d9688745e912ee293","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"90830f3da3dc3037df5e95104b75732c5207b907730dc8220cf13cb0361fb3e6","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-nxmm8x\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a86340\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-nxmm8x\" in resourcegroup=\"admin-credential-lifecycle-test-xqsgwqkm5sdb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-xqsgwqkm5sdb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-nxmm8x\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:105]: failed to begin creating HCP cluster \"admin-cred-lifecycle-nxmm8x\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a86340\u003e: \n failed starting cluster creation \"admin-cred-lifecycle-nxmm8x\" in resourcegroup=\"admin-credential-lifecycle-test-xqsgwqkm5sdb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-xqsgwqkm5sdb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-nxmm8x\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"cc8640cc2a0efad7b114699efaa464923e7fd9ebe097e4fd8a54c2e7468eb5fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"83828f42566f08f313e34932a1b6519381dc8ce91bdd407284e89ca8a43f066d","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c003e0\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-krdrw8r2xsbc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-krdrw8r2xsbc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c003e0\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-krdrw8r2xsbc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-krdrw8r2xsbc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d050a8b515f41a518888ddb64b4968cdeebaf6a79d74dfc4af8ce0a8fb71df47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"0bc9ed5ddde6d1c41f63cfc989be8d7ca185c5f36de781135291034f103b12b4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-hjdt7569xjqh/cluster-candidate-4-22-jt2n62 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ee120\u003e: \n failed to create HCP cluster cluster-candidate-4-22-jt2n62: failed starting cluster creation \"cluster-candidate-4-22-jt2n62\" in resourcegroup=\"rg-candidate-4-22-hjdt7569xjqh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hjdt7569xjqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-jt2n62\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-hjdt7569xjqh/cluster-candidate-4-22-jt2n62 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ee120\u003e: \n failed to create HCP cluster cluster-candidate-4-22-jt2n62: failed starting cluster creation \"cluster-candidate-4-22-jt2n62\" in resourcegroup=\"rg-candidate-4-22-hjdt7569xjqh\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-hjdt7569xjqh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-22-jt2n62\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d81de74d9a9e07c433b3121820995634396f96303c2e5ab886734a6c15c4e52c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"3c11fb560d81575cd3e1019f0c30dc6f8db703174863a13e3a636cb5edbeb42a","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b74760\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-qws6cjwbs4vr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-qws6cjwbs4vr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b74760\u003e: \n failed starting cluster creation \"agg-cluster\" in resourcegroup=\"feature-aggregation-qws6cjwbs4vr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-qws6cjwbs4vr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"d9934f85b1ae657c56d6ce5aaa700f3ab90166efaaaf38c9d39a511519475c30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"49de7b46b9b3907a2fe13d87c9a7bca344c3c0adfafa837d7453de185aaf1ace","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-dfk27s\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018f8260\u003e: \n failed to create HCP cluster cluster-zstream-4-22-dfk27s: failed starting cluster creation \"cluster-zstream-4-22-dfk27s\" in resourcegroup=\"rg-zstream-upgrade-4-22-bm4m4brts6tc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-bm4m4brts6tc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-dfk27s\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-dfk27s\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018f8260\u003e: \n failed to create HCP cluster cluster-zstream-4-22-dfk27s: failed starting cluster creation \"cluster-zstream-4-22-dfk27s\" in resourcegroup=\"rg-zstream-upgrade-4-22-bm4m4brts6tc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-bm4m4brts6tc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-dfk27s\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e01c905d1e26ee9eb0df566db27264ce100e316abd3c36db26ffcc9cc33df094","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"e484adf5a96f5e3e6610df6db11e90ef732c512294414f2f062b6ccb4ef4533d","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000704420\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-mvpjqnc8m77z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-mvpjqnc8m77z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000704420\u003e: \n failed to create HCP cluster negative-tests-cluster: failed starting cluster creation \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-mvpjqnc8m77z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-mvpjqnc8m77z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"e4c81e471772b3c627a23984f1111682cd6669a7a32d2744e1b3fa17d6f4df78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"2ab60864406702cac34dd6b5ff1ca75ea58460350850b988fc3ec8976ba1dc6d","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-fdskst\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108a0e0\u003e: \n failed to create HCP cluster np-noedge-fdskst: failed starting cluster creation \"np-noedge-fdskst\" in resourcegroup=\"rg-np-noedge-fdskst-pr62vdlll7tf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-fdskst-pr62vdlll7tf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-fdskst\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-fdskst\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00108a0e0\u003e: \n failed to create HCP cluster np-noedge-fdskst: failed starting cluster creation \"np-noedge-fdskst\" in resourcegroup=\"rg-np-noedge-fdskst-pr62vdlll7tf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-fdskst-pr62vdlll7tf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-fdskst\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ea079ade5ce0d8aea7335a8f02221e1353b689d246213800ceebb98c4f7b3a37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"ed979e02093bc0cb2a765eb52f1a44399359430e73772b4f7af673b82bcac847","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b72480\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7bwb7wnlxwhp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-7bwb7wnlxwhp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b72480\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed starting cluster creation \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-7bwb7wnlxwhp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-7bwb7wnlxwhp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"ee8b9fb51edd5949396aea609046c4088a17084fcd22abdb38184b116fee68f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"ea0bda361d89f660d4f4bcb7ff365c3eb3feab5621f6fe1e38c76f3ef2b369f9","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b74140\u003e: \n failed to create HCP cluster np-dg-minor-kbxpdw: failed starting cluster creation \"np-dg-minor-kbxpdw\" in resourcegroup=\"rg-np-dg-minor-kbxpdw-kb4sd88b98mn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-kbxpdw-kb4sd88b98mn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-kbxpdw\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b74140\u003e: \n failed to create HCP cluster np-dg-minor-kbxpdw: failed starting cluster creation \"np-dg-minor-kbxpdw\" in resourcegroup=\"rg-np-dg-minor-kbxpdw-kb4sd88b98mn\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-kbxpdw-kb4sd88b98mn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-kbxpdw\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f0e77ad641ba1bc5910e546955043ae925b94b135d76ca338520089ba5d9d564","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"5108d97a5606eaa075b1970aaae97cec51687933e02c22261a058ccdac27ef62","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-p6frg4\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d328c0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-p6frg4: failed starting cluster creation \"cluster-zstream-4-21-p6frg4\" in resourcegroup=\"rg-zstream-upgrade-4-21-qgrp9qmfzwh4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qgrp9qmfzwh4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-p6frg4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-p6frg4\" with version 4.21.33 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d328c0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-p6frg4: failed starting cluster creation \"cluster-zstream-4-21-p6frg4\" in resourcegroup=\"rg-zstream-upgrade-4-21-qgrp9qmfzwh4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qgrp9qmfzwh4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-p6frg4\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f3328c5389ec32d0b4d910e16bc4eff7b43ad9c2452f3520456dfca251f60cd7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"859b8e42dc2c7d0eabfe6e9a3b77d5e0f6cb1580537ab75095d267e9761de894","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010401c0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7h57t5ltz8dt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-7h57t5ltz8dt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010401c0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-7h57t5ltz8dt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-7h57t5ltz8dt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}},{"row_id":"f7b43a7961f8751a3855fd2ee9a8fcf5cad441123bf015ef3ab22b60513ad82c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2101491692665311232","occurred_at":"2026-09-20T02:00:37Z","failed_at":"e2e","signature_id":"9d43b79479d993e5a773883b2ba13571e76101d2f046d61293a9626dcea088bf","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e2e420\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e2e420\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOpenShiftVersions\n --------------------------------------------------------------------------------\n RESPONSE 502: 502 Bad Gateway\n ERROR CODE: BadGatewayConnection\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"BadGatewayConnection\",\n \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection"}}],"failure_pattern_summary":{"signature_count":56,"failure_pattern_ids":["82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54"],"matched_occurrences":56,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}