{"meta":{"date":"2026-09-20","timezone":"UTC","generated_at":"2026-10-31T08:32:27Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":36,"failed_runs":30,"runs_with_occurrences":30,"runs_with_failure_pattern_matches":27,"runs_with_unmatched_occurrences":3,"runs_without_occurrence_rows":6,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7032,"pr_state":"closed","pr_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","final_merged_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T23:42:35Z","started_at":"2026-09-20T23:42:35Z","completed_at":"2026-09-21T02:00:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T02:12:59.831769704Z","timing_metadata_checked_at":"2026-09-21T02:12:59.831769704Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T02:12:59.841473272Z","region_metadata_checked_at":"2026-09-21T02:12:59.841473272Z"},"failed_at":["alert","e2e"],"failed_test_count":17,"occurrences":[{"row_id":"02e9809cd7f8d23e2c737a00661471038d58db90188e40af0036237302688400","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"0a72c2fe9bb203f971f92d8e51c30ced5c7cc31b87aab0f0b5be2aa7c9918b6a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa01c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-rsgvv4vlx279\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"name\": \"f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:58.238504906Z\",\n \"endTime\": \"2026-09-21T00:56:02.680818108Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa01c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-rsgvv4vlx279\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"name\": \"f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:58.238504906Z\",\n \"endTime\": \"2026-09-21T00:56:02.680818108Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0b82c1db6950a781a267177aa64950fde6f6c8c501e5777f20ac6dc4e992b4b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"e8bc9a8486674bdc1a81049ad931ccbb5a9f98bfa53795eca763544cc96cde0e","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001366090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-fxwzs2\" in resourcegroup=\"rg-np-downgrade-fxwzs2-xgl2x967w267\" to finish deleting: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"name\": \"bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:01:37.424253631Z\",\n \"endTime\": \"2026-09-21T01:25:46.248853963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2nnff2dqdtbc8d6g85073gq2fnh75 still exists (deletion dispatched at 2026-09-21T01:01:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001366090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-fxwzs2\" in resourcegroup=\"rg-np-downgrade-fxwzs2-xgl2x967w267\" to finish deleting: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"name\": \"bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:01:37.424253631Z\",\n \"endTime\": \"2026-09-21T01:25:46.248853963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2nnff2dqdtbc8d6g85073gq2fnh75 still exists (deletion dispatched at 2026-09-21T01:01:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"13653feef8e0f6a8d62ace6219529068afa54cbf24b52723dad793923f9d0346","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"alert","signature_id":"5c8b752be3d2e2575305bb4467922ed388fb9c7274d746c67ce437e73fc6372a","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:59:00Z\n Ended: 2026-09-21T01:07:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2niah7ifd1s9jttm16f840o7s2bba\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2niah7ifd1s9jttm16f840o7s2bba/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T01:01:00Z\n Ended: 2026-09-21T01:07:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T01:02:03Z\n Ended: 2026-09-21T01:09:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T01:02:03Z\n Ended: 2026-09-21T01:09:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"2fc7f3ee9f94dbc32ffede670008c32d38d69a6aef96224a8d7e994f64e149c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"1d7e40ff9457b0a0fd1f510836392b2a61ea6d2ec70f11c154614ed442a91215","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006001c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nzxl7wns7x4f\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"name\": \"5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:09.310854698Z\",\n \"endTime\": \"2026-09-21T00:56:12.748107548Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006001c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nzxl7wns7x4f\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"name\": \"5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:09.310854698Z\",\n \"endTime\": \"2026-09-21T00:56:12.748107548Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b05405f46dafc8dd0ae920633fddb44c6abaf21393a6366db687ccaa33eed04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"8e12d0852cea0a6ef4c2f62124f08fedad1faab413533a1887ee7ffe0186f31d","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00079e860\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-rn5wh2crpv9m\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeede74-90f9-493d-a157-ba91fc8a2853\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeede74-90f9-493d-a157-ba91fc8a2853\",\n \"name\": \"afeede74-90f9-493d-a157-ba91fc8a2853\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:08.330592373Z\",\n \"endTime\": \"2026-09-21T00:56:12.711427367Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00079e860\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-rn5wh2crpv9m\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeede74-90f9-493d-a157-ba91fc8a2853\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/afeede74-90f9-493d-a157-ba91fc8a2853\",\n \"name\": \"afeede74-90f9-493d-a157-ba91fc8a2853\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:08.330592373Z\",\n \"endTime\": \"2026-09-21T00:56:12.711427367Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3e354663fb55940cef665b7bddd1207955cf524117dee53dcd495c5fe945cba8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"alert","signature_id":"911f13dc127a9ace065557f3d9d6b1c823b667662d3dabe8b6a218f33ca1bf47","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 24 time(s) (12 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:36:05Z\n Ended: 2026-09-21T01:14:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:37:08Z\n Ended: 2026-09-21T01:15:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T00:37:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-21T00:40:06Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-21T00:40:06Z\n Ended: 2026-09-21T01:15:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-21T00:40:06Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-21T00:41:05Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-21T00:41:05Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"41f70c76d62be6ed3c95c79b5bcb4deb2205122d96e4de3070ef772aa2f3d4d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"36da18cbc5bd0fcbf1bcfacf064639d80f3a1a5cf27962716b02e77a1bbfec85","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001226220\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ctqghmzsdxw5\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"name\": \"ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:10.309010209Z\",\n \"endTime\": \"2026-09-21T00:56:12.764905068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001226220\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ctqghmzsdxw5\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"name\": \"ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:10.309010209Z\",\n \"endTime\": \"2026-09-21T00:56:12.764905068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"583638edb772369086d3b0e1b57352af5a237f8b1c86615613d108d1eb374a5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"d6906ab33d96cae1ef1353787751cc0c7d0fd1b9e2f1ba9fdecffdd4391cd20a","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d521c0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-drf4xvsgws57\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\",\n \"name\": \"2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.910293207Z\",\n \"endTime\": \"2026-09-21T00:56:22.909745001Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d521c0\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-drf4xvsgws57\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\",\n \"name\": \"2bf17b65-e9cb-4626-b29d-dd93a3acc9c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.910293207Z\",\n \"endTime\": \"2026-09-21T00:56:22.909745001Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"63ce4156fcf1566568f31ab427bbf46d4e0aef0fb149ba8a81baf762a89792e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"5f4073b27d81b6ba979875f303e0026e5d88b19010e94d8fbda6cdd43405762d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-q2fxxd\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00161c340\u003e: \n failed to create HCP cluster cluster-zstream-5-0-q2fxxd: failed waiting for cluster=\"cluster-zstream-5-0-q2fxxd\" in resourcegroup=\"rg-zstream-upgrade-5-0-66rntshp9zkw\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfab690-d284-4689-86d1-d3a143d2d2a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfab690-d284-4689-86d1-d3a143d2d2a4\",\n \"name\": \"bcfab690-d284-4689-86d1-d3a143d2d2a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:48.427657091Z\",\n \"endTime\": \"2026-09-21T00:55:52.803861648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-q2fxxd\" with version 5.0.0-rc.1 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00161c340\u003e: \n failed to create HCP cluster cluster-zstream-5-0-q2fxxd: failed waiting for cluster=\"cluster-zstream-5-0-q2fxxd\" in resourcegroup=\"rg-zstream-upgrade-5-0-66rntshp9zkw\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfab690-d284-4689-86d1-d3a143d2d2a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfab690-d284-4689-86d1-d3a143d2d2a4\",\n \"name\": \"bcfab690-d284-4689-86d1-d3a143d2d2a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:48.427657091Z\",\n \"endTime\": \"2026-09-21T00:55:52.803861648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"66302ecd8bbb01fe9e6cde62c84746691d393007bc9ba37a80f94662326a4338","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"fb228348ca1cd58b436b3544af73f3d69eabd197936f9379767b0fcf28b2b9e2","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bcc520\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-qxqh2rljxqb9\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81fd376-89d4-498f-a01a-21c1ae71f7f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81fd376-89d4-498f-a01a-21c1ae71f7f3\",\n \"name\": \"e81fd376-89d4-498f-a01a-21c1ae71f7f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.989793193Z\",\n \"endTime\": \"2026-09-21T00:56:22.708890844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bcc520\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-qxqh2rljxqb9\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81fd376-89d4-498f-a01a-21c1ae71f7f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81fd376-89d4-498f-a01a-21c1ae71f7f3\",\n \"name\": \"e81fd376-89d4-498f-a01a-21c1ae71f7f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.989793193Z\",\n \"endTime\": \"2026-09-21T00:56:22.708890844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6bda78852f729ba5db83309e5ae946c855ff8cb7239afd63b88b7f10983c11ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"alert","signature_id":"59cd894a17dcad23b6224f622452e17a2a68039a078585dd58b1284a023c6bf0","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 24 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:46:08Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:47:08Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T00:47:09Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T00:47:09Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-21T00:47:09Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-21T00:48:11Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-21T00:48:11Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-21T00:48:11Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:16:10Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-21T00:51:12Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-21T00:51:13Z\n Ended: 2026-09-21T01:17:13Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-21T00:51:13Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-21T00:55:14Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"7d1053a7dacf27d1191a5b4c50a6f4a65a1870b33f294a87ef00cbc2c1a7e56b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"3e0062883f6a7b1aa6861d60c4689b05572197b0ccbb7b68cf2a6bb313f310ed","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7eba0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-h6stp5d8n7w6\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17c59b53-7724-4c26-9b50-9bd1833df54f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17c59b53-7724-4c26-9b50-9bd1833df54f\",\n \"name\": \"17c59b53-7724-4c26-9b50-9bd1833df54f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:54.807177783Z\",\n \"endTime\": \"2026-09-21T00:56:02.769493408Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d7eba0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-h6stp5d8n7w6\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17c59b53-7724-4c26-9b50-9bd1833df54f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17c59b53-7724-4c26-9b50-9bd1833df54f\",\n \"name\": \"17c59b53-7724-4c26-9b50-9bd1833df54f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:54.807177783Z\",\n \"endTime\": \"2026-09-21T00:56:02.769493408Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8e3d20099ff5b5be552409ff2bfe189304c5bb37544e083866e6e0e592b743cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"e126abbe3fd20bf154b2a5a83d9a532f73d8c15de2b02bdae2fe166758e207a1","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0e120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-wq4gdf4lv86c\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6274357-fa39-41af-8e56-d9301737abf7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6274357-fa39-41af-8e56-d9301737abf7\",\n \"name\": \"c6274357-fa39-41af-8e56-d9301737abf7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:48.844013158Z\",\n \"endTime\": \"2026-09-21T00:55:52.749315715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0e120\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-wq4gdf4lv86c\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6274357-fa39-41af-8e56-d9301737abf7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c6274357-fa39-41af-8e56-d9301737abf7\",\n \"name\": \"c6274357-fa39-41af-8e56-d9301737abf7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:48.844013158Z\",\n \"endTime\": \"2026-09-21T00:55:52.749315715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c1ccb14bbc156214eee6c0ac0814461afde9dd1b3342907b465dda023ed9cf65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"7423340e196d990fa25ef185f5f19b2603dd00ed381a561dc6f7f6a62639d096","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001798260\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6v4jlrzkkq4h\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40607b22-0990-4b1a-a924-bff4ba978495\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40607b22-0990-4b1a-a924-bff4ba978495\",\n \"name\": \"40607b22-0990-4b1a-a924-bff4ba978495\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:11.931014232Z\",\n \"endTime\": \"2026-09-21T00:56:12.766689996Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001798260\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-6v4jlrzkkq4h\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40607b22-0990-4b1a-a924-bff4ba978495\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/40607b22-0990-4b1a-a924-bff4ba978495\",\n \"name\": \"40607b22-0990-4b1a-a924-bff4ba978495\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:11.931014232Z\",\n \"endTime\": \"2026-09-21T00:56:12.766689996Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dece95162694e4a26ef44b56ca69cfa85e5d2c029ff18c92cd562c510863c221","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"86240fd1d6cc9a1cfec477b60417613aecc1fa5308bed9e0b396aa7862dc467a","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b963c0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d6mjtfggws4p\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc3c40a7-208d-4f13-910b-8b0045d5faab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc3c40a7-208d-4f13-910b-8b0045d5faab\",\n \"name\": \"bc3c40a7-208d-4f13-910b-8b0045d5faab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:14.850025669Z\",\n \"endTime\": \"2026-09-21T00:56:22.569668641Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b963c0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d6mjtfggws4p\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc3c40a7-208d-4f13-910b-8b0045d5faab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc3c40a7-208d-4f13-910b-8b0045d5faab\",\n \"name\": \"bc3c40a7-208d-4f13-910b-8b0045d5faab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:14.850025669Z\",\n \"endTime\": \"2026-09-21T00:56:22.569668641Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e58f63cd83b40490fb19357f0377e98c86a800a074eb36daeeb6227654b79964","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"e2e","signature_id":"4566340774f4d5644a077677c8965bc9db5a33855b406b923890c6903dd9241c","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00082c560\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-rd9wvhphr7cj\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f761987-c863-4a01-80d2-bd3d7356613c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f761987-c863-4a01-80d2-bd3d7356613c\",\n \"name\": \"7f761987-c863-4a01-80d2-bd3d7356613c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:29:35.145071349Z\",\n \"endTime\": \"2026-09-21T00:58:42.745453014Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00082c560\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-rd9wvhphr7cj\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f761987-c863-4a01-80d2-bd3d7356613c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7f761987-c863-4a01-80d2-bd3d7356613c\",\n \"name\": \"7f761987-c863-4a01-80d2-bd3d7356613c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:29:35.145071349Z\",\n \"endTime\": \"2026-09-21T00:58:42.745453014Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f2af2b309eacd7d0408fd20356814a6464f3b98ff772ef35253a4563c84ee280","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","failed_at":"alert","signature_id":"a34c4272b290720aafd3b29a745fe60b6976f6c8d6360946f5a75edf1096c9ec","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 24 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T01:01:07Z\n Ended: 2026-09-21T01:19:09Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T01:01:07Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-21T01:03:05Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:17:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-21T01:05:07Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-21T01:05:07Z\n Ended: 2026-09-21T01:17:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-21T01:05:07Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-21T01:06:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-21T01:06:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":17,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":17,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7036,"pr_state":"open","pr_sha":"ea3a7b1eaf473ee4001099d59d4391c719fb5e27","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T23:13:50Z","started_at":"2026-09-20T23:13:50Z","completed_at":"2026-09-21T02:00:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T02:12:59.946011228Z","timing_metadata_checked_at":"2026-09-21T02:12:59.946011228Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T02:12:59.950752412Z","region_metadata_checked_at":"2026-09-21T02:12:59.950752412Z"},"failed_at":["alert","e2e"],"failed_test_count":10,"occurrences":[{"row_id":"18f206f957693e37805ca819b3ae56f55cbe0dd9fd9987b685785a052f4bc7e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"b10599bdf4bc1308f14fce45589e487e1d0d085cf0ef4c4bb64b2cf8c30f79c9","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:18:44Z\n Ended: 2026-09-21T00:50:46Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4251392-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4251392-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"307d540e5306bdda70080fb027acad3479c988d7a01d001d417c1ed268281bfa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"e2e","signature_id":"ae6ec300b2e1b5ec89e4505b36ba0351611d82380af69c6bc2aa2a3220759883","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001838090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s2nb9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-2wc8bnqpjvsw\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"name\": \"3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:41.012625795Z\",\n \"endTime\": \"2026-09-21T01:06:41.448864721Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2hu8mbpcvgabffnu4e7eeji5v72ts still exists (deletion dispatched at 2026-09-21T00:42:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001838090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s2nb9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-2wc8bnqpjvsw\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"name\": \"3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:41.012625795Z\",\n \"endTime\": \"2026-09-21T01:06:41.448864721Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2hu8mbpcvgabffnu4e7eeji5v72ts still exists (deletion dispatched at 2026-09-21T00:42:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3c618356549149cc506e7521d50594e732435aa60cd4196dbda5c35f2f89b2e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"258bf6b8e2e6a15e2ce38412cd2942fddf00da1a19bafe63316ac35a44ba6d7c","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:30:39Z\n Ended: 2026-09-21T00:36:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-5q82q\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-5q82q has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"3e31181738ed75ee589a6f8117ffad835df22228825cd21f4eafce5987fecd67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"e2e","signature_id":"297f5e4b1d1f4952837a1cae50b737b42a9bc431965f0ebd876867d9c038a0d0","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100fa80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-xdxwcdw5rptk\" to finish creating: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"name\": \"f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.938506094Z\",\n \"endTime\": \"2026-09-21T00:46:21.511326736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100fa80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-xdxwcdw5rptk\" to finish creating: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"name\": \"f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.938506094Z\",\n \"endTime\": \"2026-09-21T00:46:21.511326736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"63f71d1d7a41f3919262f729189d1af4f7d04a490ad23859344b6ac9b56c4695","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"39e21beab76cee821e0ed80a6601d86d04d54351b87f382522d67356ee5fa584","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-21T01:28:40Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4251392-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5hkmr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-5hkmr (namespace arobit) on cluster ci01-j4251392-svc has averaged 126.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"aa4fdcacd8a4f86701323d44d124a044f1569a10b0ef847537ac018f296f852a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"e2e","signature_id":"eac5055c826c77b923258da473791d4fdf5723c4be718ddb37bd909244aa07ad","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e816b0\u003e: \n GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01726526-88f4-4112-af3c-fe53fa8a4dfc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01726526-88f4-4112-af3c-fe53fa8a4dfc\",\n \"name\": \"01726526-88f4-4112-af3c-fe53fa8a4dfc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:15:41.81594577Z\",\n \"endTime\": \"2026-09-21T00:44:49.261787828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e816b0\u003e: \n GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01726526-88f4-4112-af3c-fe53fa8a4dfc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/01726526-88f4-4112-af3c-fe53fa8a4dfc\",\n \"name\": \"01726526-88f4-4112-af3c-fe53fa8a4dfc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:15:41.81594577Z\",\n \"endTime\": \"2026-09-21T00:44:49.261787828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c0c81a7db0f8182ea382538c1371269fb7760f86f36c3b561ea2b28af2067a9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"a2c1fbf6e64cddf7a46eaafcc044b53f14f98a5940a39f4dbc58486310e950ce","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:49:52Z\n Ended: 2026-09-21T01:04:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"c1c9b0d6c1b5161e32c02d67d64772380d9e043b8f2c62a14cb2358483dd6a88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"386fa5f6fc68894d00954640db0f0852279a2e3a87df5f417fecec60fd94a6bb","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:35:55Z\n Ended: 2026-09-21T01:02:51Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:35:55Z\n Ended: 2026-09-21T01:06:53Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"c3ed493ac8c0bca47194ee7713e183de4467675615c56e7e290ac9691ecba846","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"alert","signature_id":"14896a8781a6fade79c483f0daff72f9c39c1157a6161055b98515a6ef455b60","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:38:51Z\n Ended: 2026-09-21T01:06:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4d11d27a-526b-40d4-812b-9a43bae7deb5\"\n Description: Pod ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"d80da60103efca7bf98933e0aae4885b139db7e5ec96b7965e39bdb976d61de1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","failed_at":"e2e","signature_id":"48429fa1e1b278e83c9c1235be2155e15cb94e91feb6a91189f07417aac2381b","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00139c1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kctnzvxjhvcd\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"name\": \"3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:02:07.101805352Z\",\n \"endTime\": \"2026-09-21T01:26:13.044715763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2i1qqe304gugno1vb23hj27er6drc still exists (deletion dispatched at 2026-09-21T01:02:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00139c1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kctnzvxjhvcd\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"name\": \"3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:02:07.101805352Z\",\n \"endTime\": \"2026-09-21T01:26:13.044715763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2i1qqe304gugno1vb23hj27er6drc still exists (deletion dispatched at 2026-09-21T01:02:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101810235566985216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7035,"pr_state":"closed","pr_sha":"18faff8c97058d15283061028cf5dd67da4bc619","final_merged_sha":"18faff8c97058d15283061028cf5dd67da4bc619","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T23:06:24Z","started_at":"2026-09-20T23:06:24Z","completed_at":"2026-09-20T23:17:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T23:35:24.284318914Z","timing_metadata_checked_at":"2026-09-20T23:35:24.284318914Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-20T23:35:24.288045202Z","region_metadata_checked_at":"2026-09-21T00:39:21.186712485Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a924fd5b7bad3d10014d8a7c100fb24a3231a372881f4791a694a399180d3779","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101810235566985216","occurred_at":"2026-09-20T23:06:24Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T23:02:29Z","started_at":"2026-09-20T23:02:29Z","completed_at":"2026-09-21T01:27:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-21T01:41:37.432073396Z","timing_metadata_checked_at":"2026-09-21T01:41:37.432073396Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-21T01:41:37.437354879Z","region_metadata_checked_at":"2026-09-21T01:41:37.437354879Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"61a2299daea1fbe35ab0d03e99cf5d7fd82287b115bedf93671f4fde23ca9211","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","failed_at":"e2e","signature_id":"e4aee9aff864d2f43ab51682459e47fa9b2962afca634816bc039aba490af5e3","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00174e0c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k6sk5ztcb6st\" to finish deleting: GET https://rp.j7913600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"name\": \"2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:34.467872497Z\",\n \"endTime\": \"2026-09-21T01:06:34.900855786Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2gfj01vbki6vfv5k1lnt4j81kjqnb still exists (deletion dispatched at 2026-09-21T00:42:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00174e0c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k6sk5ztcb6st\" to finish deleting: GET https://rp.j7913600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"name\": \"2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:34.467872497Z\",\n \"endTime\": \"2026-09-21T01:06:34.900855786Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2gfj01vbki6vfv5k1lnt4j81kjqnb still exists (deletion dispatched at 2026-09-21T00:42:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9d31c92412593ce168593ff6bfd5c137d2f457185edca3524390039b0901cc0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","failed_at":"alert","signature_id":"7d106244608ace8de6f9cae03dbed9bf2952c22134ad7882a0e61735843e27ac","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:24:26Z\n Ended: 2026-09-21T00:30:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7913600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2ggt01co6d2ehgcrfpefiv7co64vf-v20260901\", pod=\"router-fbcbc989-n4pcs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2ggt01co6d2ehgcrfpefiv7co64vf-v20260901/router-fbcbc989-n4pcs has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:24:27Z\n Ended: 2026-09-21T00:30:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7913600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2ggpos1dbpnrhtfim3df37oqp4d6v-q4q2f0t2i0p0l7c\", pod=\"router-cb77f855-wpm9t\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2ggpos1dbpnrhtfim3df37oqp4d6v-q4q2f0t2i0p0l7c/router-cb77f855-wpm9t has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c7925f2d46db8ff9bd3aa9315850ebe844918a61877de618c7ea3a8b88e94dba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","failed_at":"alert","signature_id":"f65d23db3fce211d34d0769e73e3f8f7acba651c0170fe2e1546f79d4a71b579","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:42:36Z\n Ended: 2026-09-21T00:48:35Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7913600-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7913600-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101793807640039424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7035,"pr_state":"closed","pr_sha":"18faff8c97058d15283061028cf5dd67da4bc619","final_merged_sha":"18faff8c97058d15283061028cf5dd67da4bc619","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T22:01:03Z","started_at":"2026-09-20T22:01:03Z","completed_at":"2026-09-20T22:10:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:20:34.218887245Z","timing_metadata_checked_at":"2026-09-20T22:20:34.218887245Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-20T22:20:34.221789635Z","region_metadata_checked_at":"2026-09-20T23:24:25.230688459Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5f8a171339e4fa4b01cb33f8a405fb8e94d088d0b6faaf89b8949f02006768a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101793807640039424","occurred_at":"2026-09-20T22:01:03Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101793797569515520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7032,"pr_state":"closed","pr_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","final_merged_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T22:00:55Z","started_at":"2026-09-20T22:00:55Z","completed_at":"2026-09-20T22:10:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:20:34.035755665Z","timing_metadata_checked_at":"2026-09-20T22:20:34.035755665Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-20T22:20:34.039113153Z","region_metadata_checked_at":"2026-09-20T23:24:25.10530806Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"07eb0d0cf2b8480cb9a23375c12127b091d5b39fbbe3335b3909f8463082131e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101793797569515520","occurred_at":"2026-09-20T22:00:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101767145149960192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7033,"pr_state":"closed","pr_sha":"c448a0db20d77ea047c00534044b12cc2b2ac565","final_merged_sha":"ef821496d20065a3db001282a41d83827d994051","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T20:15:04Z","started_at":"2026-09-20T20:15:04Z","completed_at":"2026-09-20T22:20:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:31:19.992970333Z","timing_metadata_checked_at":"2026-09-20T22:31:19.992970333Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T22:31:19.995885724Z","region_metadata_checked_at":"2026-09-20T22:31:19.995885724Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4fb4380e15dfd4cd63886dfa7fbfc67d4248bb08efd96f7c38b97cd4da166065","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101767145149960192","occurred_at":"2026-09-20T20:15:04Z","failed_at":"e2e","signature_id":"55fec78f783ce8cf181e98d25b36b712ff6eab60ce5978dd2cbcd0aee43a639c","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001dac330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lfwqsfxv46jf\" to finish deleting: GET https://rp.j9960192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11bbccb6-f070-4edf-a07d-e5d2b5dbade5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11bbccb6-f070-4edf-a07d-e5d2b5dbade5\",\n \"name\": \"11bbccb6-f070-4edf-a07d-e5d2b5dbade5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:32:16.596553994Z\",\n \"endTime\": \"2026-09-20T21:56:23.901016482Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvlb6tjn805avjlrq54um1k9ghb2e1 still exists (deletion dispatched at 2026-09-20T21:32:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001dac330\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lfwqsfxv46jf\" to finish deleting: GET https://rp.j9960192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11bbccb6-f070-4edf-a07d-e5d2b5dbade5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/11bbccb6-f070-4edf-a07d-e5d2b5dbade5\",\n \"name\": \"11bbccb6-f070-4edf-a07d-e5d2b5dbade5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:32:16.596553994Z\",\n \"endTime\": \"2026-09-20T21:56:23.901016482Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvlb6tjn805avjlrq54um1k9ghb2e1 still exists (deletion dispatched at 2026-09-20T21:32:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f4197d0de3962b76d7915e149600df79d9d5574af1a8b654b80991c16c6a2df3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101767145149960192","occurred_at":"2026-09-20T20:15:04Z","failed_at":"e2e","signature_id":"753848c3bc300eb5da8ad64aa08adc3a8b8dafee7fb505106b6f5cbf501ef653","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011c8078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-c465zc8xkqx7\" to finish deleting: GET https://rp.j9960192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ec0221a2-95f2-4368-b530-97209abb26ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ec0221a2-95f2-4368-b530-97209abb26ab\",\n \"name\": \"ec0221a2-95f2-4368-b530-97209abb26ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:41:10.549898621Z\",\n \"endTime\": \"2026-09-20T22:05:13.934972715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvm850d5khb2asjdadkcgoks2iqobg still exists (deletion dispatched at 2026-09-20T21:41:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011c8078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-c465zc8xkqx7\" to finish deleting: GET https://rp.j9960192.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ec0221a2-95f2-4368-b530-97209abb26ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ec0221a2-95f2-4368-b530-97209abb26ab\",\n \"name\": \"ec0221a2-95f2-4368-b530-97209abb26ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:41:10.549898621Z\",\n \"endTime\": \"2026-09-20T22:05:13.934972715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvm850d5khb2asjdadkcgoks2iqobg still exists (deletion dispatched at 2026-09-20T21:41:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T19:59:59Z","started_at":"2026-09-20T19:59:59Z","completed_at":"2026-09-20T22:01:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:10:34.004824246Z","timing_metadata_checked_at":"2026-09-20T22:10:34.004824246Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T22:10:34.009069832Z","region_metadata_checked_at":"2026-09-20T22:10:34.009069832Z"},"failed_at":["alert","e2e"],"failed_test_count":7,"occurrences":[{"row_id":"2ab909dd4e9b4f90b480bb53ab5308e59e6dbd540d30ccbb5717b9b15410e23c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"571f2e2e0cd4941242495f0f83848c90e92c491345357a2fc6f06f0266aa0a09","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef5080\u003e: \n DNS for route host agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef5080\u003e: \n DNS for route host agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"40c5c3ae1a4dd2e4450450ab1130526b102a6263cd4db10ae8beb349e5923494","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"ae135f0135bd68e7f6075a32b2c72f3bdce2c27a9f86f02269bf9ec0d9c29b8b","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002510348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-w9lxmqgrgjxc\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\",\n \"name\": \"d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:00:38.791668601Z\",\n \"endTime\": \"2026-09-20T21:24:44.133089654Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvaqju2imrg5degq03jceicg07q1sk still exists (deletion dispatched at 2026-09-20T21:00:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002510348\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-w9lxmqgrgjxc\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\",\n \"name\": \"d14c5a49-e2cb-44e8-9f11-72dd4ad99b35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:00:38.791668601Z\",\n \"endTime\": \"2026-09-20T21:24:44.133089654Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvaqju2imrg5degq03jceicg07q1sk still exists (deletion dispatched at 2026-09-20T21:00:39Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"82b02cb558491a0dfcb05a6312dc71ada50ccd9673e699b6e60a63f70a348374","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"1de82b4d07b3d460f0773709ee7a723eb163f891926727a6696a1f16b19d90b5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015b00f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-x24g9c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-zwzt7v9n4qld\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d039276d-bda2-483a-a789-67d1043bffad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d039276d-bda2-483a-a789-67d1043bffad\",\n \"name\": \"d039276d-bda2-483a-a789-67d1043bffad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:01:45.790091965Z\",\n \"endTime\": \"2026-09-20T21:25:54.10177225Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvajhucng7kcjeavelijr1m7gh4dvh still exists (deletion dispatched at 2026-09-20T21:01:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015b00f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-x24g9c\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-zwzt7v9n4qld\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d039276d-bda2-483a-a789-67d1043bffad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d039276d-bda2-483a-a789-67d1043bffad\",\n \"name\": \"d039276d-bda2-483a-a789-67d1043bffad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:01:45.790091965Z\",\n \"endTime\": \"2026-09-20T21:25:54.10177225Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvajhucng7kcjeavelijr1m7gh4dvh still exists (deletion dispatched at 2026-09-20T21:01:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"84fe011b53eefdd61e326845fe8c39c49cb1074ea1ddc8276bc312f378bff940","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"6b62ee5741da84d36547bbd46f0d60714cb4fa8627bb19d966f92a3eba03e28a","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0018b60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-gmw544q97wff\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df6c4988-d57f-4cfa-89da-42c4c8598408\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df6c4988-d57f-4cfa-89da-42c4c8598408\",\n \"name\": \"df6c4988-d57f-4cfa-89da-42c4c8598408\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:20:32.391377317Z\",\n \"endTime\": \"2026-09-20T21:44:34.405759905Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvarbr3muag22o8gt4v9kc9nb03f6d still exists (deletion dispatched at 2026-09-20T21:20:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0018b60f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-gmw544q97wff\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df6c4988-d57f-4cfa-89da-42c4c8598408\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/df6c4988-d57f-4cfa-89da-42c4c8598408\",\n \"name\": \"df6c4988-d57f-4cfa-89da-42c4c8598408\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:20:32.391377317Z\",\n \"endTime\": \"2026-09-20T21:44:34.405759905Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvarbr3muag22o8gt4v9kc9nb03f6d still exists (deletion dispatched at 2026-09-20T21:20:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9582bd4cd943f84a97b8a37fbf59f6d65e52be2f5249fc051e1787508a7ad991","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"35c9e2930b67189ab544a31c2246d603785a353beb1727154337c02311b5ba68","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001846078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-8xl9z7\" in resourcegroup=\"rg-zstream-upgrade-4-21-8m66bndkns9t\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3409ee9c-20cd-4a04-bbea-059ca801b95b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3409ee9c-20cd-4a04-bbea-059ca801b95b\",\n \"name\": \"3409ee9c-20cd-4a04-bbea-059ca801b95b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:01:55.419339651Z\",\n \"endTime\": \"2026-09-20T21:26:04.130585156Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvahv650fl61a4dvgvtu796fkako2j still exists (deletion dispatched at 2026-09-20T21:02:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001846078\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-8xl9z7\" in resourcegroup=\"rg-zstream-upgrade-4-21-8m66bndkns9t\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3409ee9c-20cd-4a04-bbea-059ca801b95b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3409ee9c-20cd-4a04-bbea-059ca801b95b\",\n \"name\": \"3409ee9c-20cd-4a04-bbea-059ca801b95b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:01:55.419339651Z\",\n \"endTime\": \"2026-09-20T21:26:04.130585156Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvahv650fl61a4dvgvtu796fkako2j still exists (deletion dispatched at 2026-09-20T21:02:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d2d359344b56f2e35dc32c77172eebf32fe5bc0e3f1e55a7e3faf27395528500","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"alert","signature_id":"8f4619349028d8271e222f8d6571b2f090f7d7878ecb45ed7f028937641f406f","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T21:05:56Z\n Ended: 2026-09-20T21:11:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8645120-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8645120-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"e563a12983d7f7f9811f114d67585d0c17e21f3dcdb0a71f8e6024c18aded0e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","failed_at":"e2e","signature_id":"6ecc7b9cabb4ec501df9af223046a31e6f46790bc97afea6159cc8363fead315","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001418108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-zdswd5\" in resourcegroup=\"rg-zstream-upgrade-4-20-xsntgddnxfll\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ba72035-9acd-4b81-9004-66ff1293e676\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ba72035-9acd-4b81-9004-66ff1293e676\",\n \"name\": \"3ba72035-9acd-4b81-9004-66ff1293e676\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:05:10.531590012Z\",\n \"endTime\": \"2026-09-20T21:29:14.121060804Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvamm4ck5qen7rpohu5h53k3sludes still exists (deletion dispatched at 2026-09-20T21:05:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001418108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-zdswd5\" in resourcegroup=\"rg-zstream-upgrade-4-20-xsntgddnxfll\" to finish deleting: GET https://rp.j8645120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ba72035-9acd-4b81-9004-66ff1293e676\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3ba72035-9acd-4b81-9004-66ff1293e676\",\n \"name\": \"3ba72035-9acd-4b81-9004-66ff1293e676\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:05:10.531590012Z\",\n \"endTime\": \"2026-09-20T21:29:14.121060804Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvamm4ck5qen7rpohu5h53k3sludes still exists (deletion dispatched at 2026-09-20T21:05:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101762182411718656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7035,"pr_state":"closed","pr_sha":"18faff8c97058d15283061028cf5dd67da4bc619","final_merged_sha":"18faff8c97058d15283061028cf5dd67da4bc619","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T19:55:27Z","started_at":"2026-09-20T19:55:27Z","completed_at":"2026-09-20T21:45:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:00:09.160560343Z","timing_metadata_checked_at":"2026-09-20T22:00:09.160560343Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T22:00:09.164119432Z","region_metadata_checked_at":"2026-09-20T22:00:09.164119432Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6bc834e630c0a71f4bddc3195d298b758f1630f57278854ea502bfbb588474dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101762182411718656","occurred_at":"2026-09-20T19:55:27Z","failed_at":"alert","signature_id":"993e0edbc2802d27b58b7315a4badc5470fe6efc5c6616983981032abe8b49db","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T21:15:25Z\n Ended: 2026-09-20T21:27:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j1718656-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j1718656-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7030,"pr_state":"closed","pr_sha":"4dc2c443d9d2f9fa2a598585253f8231c7ddf46c","final_merged_sha":"50f9710d6eb949ad5cf7c325f7c4d6cce9083ddf","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T19:26:34Z","started_at":"2026-09-20T19:26:34Z","completed_at":"2026-09-20T22:02:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T22:10:34.12856235Z","timing_metadata_checked_at":"2026-09-20T22:10:34.12856235Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T22:10:34.13160584Z","region_metadata_checked_at":"2026-09-20T22:10:34.13160584Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"04ffcaf7762f47ecf73e0578e98552f5ce37fe12b44c37983f7dca3c8ba7c9f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","failed_at":"alert","signature_id":"e7c54520064dc8897e885bf4790b5cdcc177d7e87c6cdbf61692ea5647d049d2","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T21:06:19Z\n Ended: 2026-09-20T21:13:22Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7010688-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7010688-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"42a23fe7038468a62121df425e6e6a85251f017045c5cfb333451071e07732f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","failed_at":"e2e","signature_id":"94c88bfe25b6cc5073858901db97efc9bfdacd7a00fdfffa45d0abf7a6f10cff","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00122c0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-wkvn4m8rs9f7\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c840051-bc7e-4300-9c8b-cdc5ba02f524\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c840051-bc7e-4300-9c8b-cdc5ba02f524\",\n \"name\": \"5c840051-bc7e-4300-9c8b-cdc5ba02f524\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:21:26.975187286Z\",\n \"endTime\": \"2026-09-20T21:45:29.127892727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvbff7c84adptpfq9s8m0tpv7v5ddl still exists (deletion dispatched at 2026-09-20T21:21:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00122c0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-wkvn4m8rs9f7\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c840051-bc7e-4300-9c8b-cdc5ba02f524\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5c840051-bc7e-4300-9c8b-cdc5ba02f524\",\n \"name\": \"5c840051-bc7e-4300-9c8b-cdc5ba02f524\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:21:26.975187286Z\",\n \"endTime\": \"2026-09-20T21:45:29.127892727Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvbff7c84adptpfq9s8m0tpv7v5ddl still exists (deletion dispatched at 2026-09-20T21:21:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"51401fbe99f3cf3dd11c2ce463f61972b770f7fcc35f4cd2de936f121e2072bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","failed_at":"e2e","signature_id":"ba4ec05dffbf7aeb4b673f4a633bc6bb64ed25fa0b09ea36f4467a08976b4236","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a7c258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-8wrhrq\" in resourcegroup=\"rg-zstream-upgrade-4-22-sbnm9pqkcwf6\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\",\n \"name\": \"2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:02:20.578949773Z\",\n \"endTime\": \"2026-09-20T21:26:20.682855256Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvahe0tlai0bmukqovr96jjrqs3mbm still exists (deletion dispatched at 2026-09-20T21:02:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a7c258\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-8wrhrq\" in resourcegroup=\"rg-zstream-upgrade-4-22-sbnm9pqkcwf6\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\",\n \"name\": \"2e89dceb-0d9e-4486-8c92-d0f7c5bd518a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:02:20.578949773Z\",\n \"endTime\": \"2026-09-20T21:26:20.682855256Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvahe0tlai0bmukqovr96jjrqs3mbm still exists (deletion dispatched at 2026-09-20T21:02:22Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"88842db398506118e17a77b985575c009ec44fea0a9fcda307afe67f4b3c8c2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","failed_at":"e2e","signature_id":"5e8455d974423d74da7a2764a6de9171d767628d73b6837a9e7152a9dc31c031","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016aa0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-76bn77\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-8cjn4vhljp95\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddd0e4ac-a249-441c-b385-574e70d538ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddd0e4ac-a249-441c-b385-574e70d538ac\",\n \"name\": \"ddd0e4ac-a249-441c-b385-574e70d538ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:02:37.113406649Z\",\n \"endTime\": \"2026-09-20T21:26:40.645444433Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvagov3s5e6i263h39l67869g6anai still exists (deletion dispatched at 2026-09-20T21:02:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016aa0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-76bn77\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-8cjn4vhljp95\" to finish deleting: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddd0e4ac-a249-441c-b385-574e70d538ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ddd0e4ac-a249-441c-b385-574e70d538ac\",\n \"name\": \"ddd0e4ac-a249-441c-b385-574e70d538ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T21:02:37.113406649Z\",\n \"endTime\": \"2026-09-20T21:26:40.645444433Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suvagov3s5e6i263h39l67869g6anai still exists (deletion dispatched at 2026-09-20T21:02:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d48ecf48fc0366e4f72b9887aa18aac2c837aeebe1466f6afaba7d9289ee5693","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","failed_at":"e2e","signature_id":"fec305f359038a8b537a558ba87b3d6d7b2c427d51eb493f15dad5445be91e77","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c6e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-hxgb2rxvk82d\" to finish creating: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"name\": \"2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:48:17.088169395Z\",\n \"endTime\": \"2026-09-20T21:07:20.582868448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c6e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-hxgb2rxvk82d\" to finish creating: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"name\": \"2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:48:17.088169395Z\",\n \"endTime\": \"2026-09-20T21:07:20.582868448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7032,"pr_state":"closed","pr_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","final_merged_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T19:01:41Z","started_at":"2026-09-20T19:01:41Z","completed_at":"2026-09-20T21:18:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T21:28:28.34717848Z","timing_metadata_checked_at":"2026-09-20T21:28:28.34717848Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T21:28:28.350753869Z","region_metadata_checked_at":"2026-09-20T21:28:28.350753869Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"1b8ffee88db2d9644f214d173b10667549564aa79636c133c0bb85385e5386e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"204e1750cfbcdccc7a1aa87cf8b29089bd7645257cbc1509ff43f3d1c64bf5f6","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:518]: Timed out after 900.001s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:505 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: ...\nto be \u003e=\n \u003cint\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:518]: Timed out after 900.001s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:505 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: ...\nto be \u003e=\n \u003cint\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have at least 2 nodes (autoscaling min), found 0","search_query":"node pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0"}},{"row_id":"251bab08c74f242248bf8b408cbe65e12cc630f8f6eb4fa5426fd6c014c4cba4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"08debcb95fe76787cc5e7470c732a78966ba147069573bb898c775a88f1cff3e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000219dc0\u003e: \n failed to create NodePool np-ggj24g: failed waiting for nodepool=\"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\" in resourcegroup=\"rg-candidate-4-20-686sncjvb2sx\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"name\": \"60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:06:27.477664357Z\",\n \"endTime\": \"2026-09-20T20:25:30.377289875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000219dc0\u003e: \n failed to create NodePool np-ggj24g: failed waiting for nodepool=\"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\" in resourcegroup=\"rg-candidate-4-20-686sncjvb2sx\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"name\": \"60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:06:27.477664357Z\",\n \"endTime\": \"2026-09-20T20:25:30.377289875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b3810171f30121cd8713e3d56756d3faec14cf7125baae340883643c8ad6a1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"48929c5f392dda19c39d764a9d4117d26ea3b70f835a97462a0249c92cc9fdf4","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"422ddc32f327db7a61c881b4411ef7767ec79e98b46476bd6a0cffdccf708740","failure_pattern":"expected all nodes to have label 'key1=value1'","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'"}},{"row_id":"55d7f598d1a41824505461f5e95aaf17529ca46721083fb901c92de616f0db9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade"}},{"row_id":"594547218134bd8028a5e91412c7f97d36bb85dd140aad97e46b8b6d20fd6422","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"f61d48f7e4dfc0d5ed670d4815de996c6e11f3ee6013821737755d17d338fd53","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010007a0\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010007a0\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"55fc315450b35ec48de3526d899619e34f6aae1d138260064fe2d53d5403f86e","failure_pattern":"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet \u003cnamespace\u003e/global-pull-secret-syncer has no desired pods scheduled","search_query":"occurred"}},{"row_id":"84af0e38f504268297a3db4476d8fd473b54efa2fa564c726e527b345e479498","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"e54b973f5d0dfed8acc1e8e3583260026a6a968d1ae6ee942caf592be98feaeb","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe67a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-9p6vvl\" in resourcegroup=\"rg-np-noedge-9p6vvl-v6jt8npv8wnh\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3713b0f0-249d-4350-9997-27de0131dfcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3713b0f0-249d-4350-9997-27de0131dfcd\",\n \"name\": \"3713b0f0-249d-4350-9997-27de0131dfcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:02:41.432631359Z\",\n \"endTime\": \"2026-09-20T20:21:50.467881337Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe67a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-9p6vvl\" in resourcegroup=\"rg-np-noedge-9p6vvl-v6jt8npv8wnh\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3713b0f0-249d-4350-9997-27de0131dfcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3713b0f0-249d-4350-9997-27de0131dfcd\",\n \"name\": \"3713b0f0-249d-4350-9997-27de0131dfcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:02:41.432631359Z\",\n \"endTime\": \"2026-09-20T20:21:50.467881337Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9eab0a1eddffa2e0aecd4cf75a6ac1524838cc900ecd0dce5d321c8fb6c6f950","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"db98f57638dcde7ae6924500ef05678fc197bea0143f095eee401504ac389cb8","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00199a0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-mr4hsq\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-tl7tb6ttppj7\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9eafb140-dd1b-4c27-a86b-1a1997869e11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9eafb140-dd1b-4c27-a86b-1a1997869e11\",\n \"name\": \"9eafb140-dd1b-4c27-a86b-1a1997869e11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:20:56.231911113Z\",\n \"endTime\": \"2026-09-20T20:45:00.42480766Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suungo1m007b7a8qlj4ggi47d944qra still exists (deletion dispatched at 2026-09-20T20:20:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00199a0d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-mr4hsq\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-tl7tb6ttppj7\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9eafb140-dd1b-4c27-a86b-1a1997869e11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9eafb140-dd1b-4c27-a86b-1a1997869e11\",\n \"name\": \"9eafb140-dd1b-4c27-a86b-1a1997869e11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:20:56.231911113Z\",\n \"endTime\": \"2026-09-20T20:45:00.42480766Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suungo1m007b7a8qlj4ggi47d944qra still exists (deletion dispatched at 2026-09-20T20:20:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aa4dcd6f2765b96ab3c2fd68d918016ea1ed6c40dac4a365a21fd17d85456c4b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"81d3fa9d4a49bf594db48e69cfb405302d6d567f873fa4a49e7a724e1f6a4553","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a82108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-vctfds\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-kw9j54cmg4pc\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4e37d117-e332-4860-a681-21465cb9fec9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4e37d117-e332-4860-a681-21465cb9fec9\",\n \"name\": \"4e37d117-e332-4860-a681-21465cb9fec9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:23:12.755096811Z\",\n \"endTime\": \"2026-09-20T20:47:20.418777117Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suunh30duok5htrpqis3a229ruh2gu8 still exists (deletion dispatched at 2026-09-20T20:23:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a82108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-vctfds\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-kw9j54cmg4pc\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4e37d117-e332-4860-a681-21465cb9fec9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4e37d117-e332-4860-a681-21465cb9fec9\",\n \"name\": \"4e37d117-e332-4860-a681-21465cb9fec9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:23:12.755096811Z\",\n \"endTime\": \"2026-09-20T20:47:20.418777117Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suunh30duok5htrpqis3a229ruh2gu8 still exists (deletion dispatched at 2026-09-20T20:23:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c20ff8783765f263971647de32041ce98788190a9255de24759765697c8d22be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"531647f22030c4eb57aa8d0ce0f1192ad57dde45a170d15dbfbc6d84af797341","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dfc100\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-wbdbqt\" in resourcegroup=\"rg-np-version-upgrade-wbdbqt-c4kwcbz5jpzk\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b9c3e353-8a9e-429f-b72a-d1506b35ff09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b9c3e353-8a9e-429f-b72a-d1506b35ff09\",\n \"name\": \"b9c3e353-8a9e-429f-b72a-d1506b35ff09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:03:32.070997575Z\",\n \"endTime\": \"2026-09-20T20:22:40.415779239Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dfc100\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-wbdbqt\" in resourcegroup=\"rg-np-version-upgrade-wbdbqt-c4kwcbz5jpzk\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b9c3e353-8a9e-429f-b72a-d1506b35ff09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b9c3e353-8a9e-429f-b72a-d1506b35ff09\",\n \"name\": \"b9c3e353-8a9e-429f-b72a-d1506b35ff09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:03:32.070997575Z\",\n \"endTime\": \"2026-09-20T20:22:40.415779239Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cb9dc326990e2c6c26ac44245386ef91018e5f17d25ea90d03967f27567362f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"de84625bbe06b7fedfd12cf6c287f90f560af49bcb2f31d73c8b14a307cfe92d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:474]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:474]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"e22addc8ad702d24529905c2d8ecae8bc9e8ecb5f14bedb2fa4043c989ebd7b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","failed_at":"e2e","signature_id":"6430a6d078ca6a6f372c5a118c728de07a27f33f7e2dd5e2fdf9a7c60c12d6f7","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00126e0f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-4rwkxxhrqmpn\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/67887784-f8ef-49d8-9a3a-e8734d5e320e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/67887784-f8ef-49d8-9a3a-e8734d5e320e\",\n \"name\": \"67887784-f8ef-49d8-9a3a-e8734d5e320e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:38:02.27895881Z\",\n \"endTime\": \"2026-09-20T21:02:10.843079298Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suunh6uiggfn6q1bvsp3td8abp45hh0 still exists (deletion dispatched at 2026-09-20T20:38:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00126e0f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-4rwkxxhrqmpn\" to finish deleting: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/67887784-f8ef-49d8-9a3a-e8734d5e320e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/67887784-f8ef-49d8-9a3a-e8734d5e320e\",\n \"name\": \"67887784-f8ef-49d8-9a3a-e8734d5e320e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:38:02.27895881Z\",\n \"endTime\": \"2026-09-20T21:02:10.843079298Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suunh6uiggfn6q1bvsp3td8abp45hh0 still exists (deletion dispatched at 2026-09-20T20:38:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["422ddc32f327db7a61c881b4411ef7767ec79e98b46476bd6a0cffdccf708740","55fc315450b35ec48de3526d899619e34f6aae1d138260064fe2d53d5403f86e","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T17:10:13Z","started_at":"2026-09-20T17:10:13Z","completed_at":"2026-09-20T19:14:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T19:23:26.956472331Z","timing_metadata_checked_at":"2026-09-20T19:23:26.956472331Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T19:23:26.959900721Z","region_metadata_checked_at":"2026-09-20T19:23:26.959900721Z"},"failed_at":["alert","e2e"],"failed_test_count":15,"occurrences":[{"row_id":"0122f5826abd42c43d62e9f73009fb5437a4c3ba79541255dd70929cb08074bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"alert","signature_id":"5e756453c54bd40ab2d5dd229f82f628f7919ef41eabbc02f3d959659f4f7234","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 20 time(s) (10 unknown, 10 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-20T18:00:47Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-20T18:01:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"2185a07f705f9f3fadeea684cf0d4241fd07abdfbdd7f67261fa1ac69542e62f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"0122ecc6da0d7f29528efc4c03aacfe2a95abd22053ee427b3ba034f89f24578","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012a4340\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-lzrwjdpxp8w2\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b173996-7453-4b97-b67f-ef797fded833\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b173996-7453-4b97-b67f-ef797fded833\",\n \"name\": \"6b173996-7453-4b97-b67f-ef797fded833\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:01.158204247Z\",\n \"endTime\": \"2026-09-20T18:19:10.424114875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012a4340\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-lzrwjdpxp8w2\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b173996-7453-4b97-b67f-ef797fded833\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b173996-7453-4b97-b67f-ef797fded833\",\n \"name\": \"6b173996-7453-4b97-b67f-ef797fded833\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:01.158204247Z\",\n \"endTime\": \"2026-09-20T18:19:10.424114875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3175ba2f730a74105a2c93536a32d55a63f01b5e74618ec26e64c9d1604344dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"bf5a3ba4ae1888ec5bbf4c1ea6088363a29b33944bb08384b179d0e7edc222cb","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d64220\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-d5dfblc9jxtp\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa297b6b-fe8e-4087-b659-ba08e6f85c73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa297b6b-fe8e-4087-b659-ba08e6f85c73\",\n \"name\": \"aa297b6b-fe8e-4087-b659-ba08e6f85c73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:51:22.937689743Z\",\n \"endTime\": \"2026-09-20T18:20:30.411667073Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d64220\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-d5dfblc9jxtp\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa297b6b-fe8e-4087-b659-ba08e6f85c73\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa297b6b-fe8e-4087-b659-ba08e6f85c73\",\n \"name\": \"aa297b6b-fe8e-4087-b659-ba08e6f85c73\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:51:22.937689743Z\",\n \"endTime\": \"2026-09-20T18:20:30.411667073Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"47f9005bee847c29df7beaebcb5920253cf086a2102dc9dbbf3b1e84e0c16db1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"e6538054ab0559cddb2c0ed63235c48a4d5cd57bef25e042c4c66fb7e93ba92e","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c0f60\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2fppphk96nck\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"name\": \"648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:38.502314717Z\",\n \"endTime\": \"2026-09-20T18:19:40.439163186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c0f60\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2fppphk96nck\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"name\": \"648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:38.502314717Z\",\n \"endTime\": \"2026-09-20T18:19:40.439163186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"44fc5c170ee5969da686dbaeb61a39c91b4e9f01cf7a4324b16151d7f8a7811a","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4c22a1b01844201cfcfdc984f4b575d79c5ea412e8f882ddf242bacf6042887a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"a2be2cf8041303505466850a3cb9c21d1f99bbb048308b395ef9642bc9197ca2","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6c600\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-gc4d4nn6mhwj\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7556a704-973b-41e7-a11a-80eff56506d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7556a704-973b-41e7-a11a-80eff56506d4\",\n \"name\": \"7556a704-973b-41e7-a11a-80eff56506d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:01.159372023Z\",\n \"endTime\": \"2026-09-20T18:19:10.475238514Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6c600\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-gc4d4nn6mhwj\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7556a704-973b-41e7-a11a-80eff56506d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7556a704-973b-41e7-a11a-80eff56506d4\",\n \"name\": \"7556a704-973b-41e7-a11a-80eff56506d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:01.159372023Z\",\n \"endTime\": \"2026-09-20T18:19:10.475238514Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4d9776443d827be3c6e37637e5df0c2abec7249241c02712046df48bf05ae55e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"alert","signature_id":"b4c5714662b6eebf9aeab7aa2383a89d5f76557fa6ae1c0f2264a9d31c883f89","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-20T18:55:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3380352-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qgqnm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-qgqnm (namespace arobit) on cluster ci01-j3380352-svc has averaged 126.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"6548e68cfb1ff1fb19544ccfcc21aee5769c5c07ab8a8e3dc07e7de01b8dff0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"66049a9e3aec5154c649a5163e0d3ee1db514ce22e0071326628d432e4a129e1","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00138a198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-x79rddkrff4p\" to finish deleting: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5321d526-4d45-49af-b781-508e5635a670\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5321d526-4d45-49af-b781-508e5635a670\",\n \"name\": \"5321d526-4d45-49af-b781-508e5635a670\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T18:34:21.423041234Z\",\n \"endTime\": \"2026-09-20T18:58:28.878805957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sustfjvol3npjlb5trmhmvn0qgrv50a still exists (deletion dispatched at 2026-09-20T18:34:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00138a198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-x79rddkrff4p\" to finish deleting: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5321d526-4d45-49af-b781-508e5635a670\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5321d526-4d45-49af-b781-508e5635a670\",\n \"name\": \"5321d526-4d45-49af-b781-508e5635a670\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T18:34:21.423041234Z\",\n \"endTime\": \"2026-09-20T18:58:28.878805957Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sustfjvol3npjlb5trmhmvn0qgrv50a still exists (deletion dispatched at 2026-09-20T18:34:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6c3ddb6263f49fc3f8d369e07efe10154c44e91bf4381fdf9ef73e1170f109cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"6bc7534690e6bb03e8cc5f05e1f2bfee87b13f961ed915d568dd8e6b92f0e0f2","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a56a60\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8wclt4mlsd89\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc19e435-ddc8-468a-83ba-b4ec238ff165\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc19e435-ddc8-468a-83ba-b4ec238ff165\",\n \"name\": \"bc19e435-ddc8-468a-83ba-b4ec238ff165\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:58.826266654Z\",\n \"endTime\": \"2026-09-20T18:19:00.407536942Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a56a60\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8wclt4mlsd89\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc19e435-ddc8-468a-83ba-b4ec238ff165\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc19e435-ddc8-468a-83ba-b4ec238ff165\",\n \"name\": \"bc19e435-ddc8-468a-83ba-b4ec238ff165\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:58.826266654Z\",\n \"endTime\": \"2026-09-20T18:19:00.407536942Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"75e647cb74cc1b5f69e5e1695080def5d27fc40d0a9c28a957bd70da7446fa8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"aea772e72e8fe85891b43563346768f4449021077f900a91ce91ffe2eb62aef3","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00181a220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-xth4kfdgl9qd\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84362293-473a-4529-8480-825894137a63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84362293-473a-4529-8480-825894137a63\",\n \"name\": \"84362293-473a-4529-8480-825894137a63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:49.637399572Z\",\n \"endTime\": \"2026-09-20T18:18:50.472292068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00181a220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-xth4kfdgl9qd\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84362293-473a-4529-8480-825894137a63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84362293-473a-4529-8480-825894137a63\",\n \"name\": \"84362293-473a-4529-8480-825894137a63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:49.637399572Z\",\n \"endTime\": \"2026-09-20T18:18:50.472292068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"98057eac6d67991ec7317fadc3faa2bb273575ab3c78370d97be2119ff2eabff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"alert","signature_id":"8da3d9c87ac6e15f84a876a8ec0eb6d135540c742388bef66277cd5947ec5e3b","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 20 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:28:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:26:53Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-20T18:10:51Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-20T18:10:51Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-20T18:10:51Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-20T18:10:52Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"9c766dd07bf1d4cf3ad48907cf7c9e72d78833a1502e0375573da2c5e97ae1dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"c4e33f88c11b927f53f19ef5f88b383cbdc6921ffa0d1b5ed16a016693d69ee1","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000888600\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-d7ffrxgdklfr\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54b46b89-b4a9-4ae9-b551-78a76e071ebf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54b46b89-b4a9-4ae9-b551-78a76e071ebf\",\n \"name\": \"54b46b89-b4a9-4ae9-b551-78a76e071ebf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:02.573105706Z\",\n \"endTime\": \"2026-09-20T18:19:10.459388953Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000888600\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-d7ffrxgdklfr\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54b46b89-b4a9-4ae9-b551-78a76e071ebf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54b46b89-b4a9-4ae9-b551-78a76e071ebf\",\n \"name\": \"54b46b89-b4a9-4ae9-b551-78a76e071ebf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:02.573105706Z\",\n \"endTime\": \"2026-09-20T18:19:10.459388953Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9d5a5b5375c0faa5ae269008f3d10cb24a7bf5d7768776210f273bcc750e9a7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"a682bdcc6ac5308107646bacc01ed06ff73822f3817c828a7af6eeb2d50dbb37","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097f060\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-mpqhctkn8kzf\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f0788d5f-c399-4ffb-8320-8f18588385ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f0788d5f-c399-4ffb-8320-8f18588385ec\",\n \"name\": \"f0788d5f-c399-4ffb-8320-8f18588385ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:48.534799023Z\",\n \"endTime\": \"2026-09-20T18:18:50.467299699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097f060\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-mpqhctkn8kzf\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f0788d5f-c399-4ffb-8320-8f18588385ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f0788d5f-c399-4ffb-8320-8f18588385ec\",\n \"name\": \"f0788d5f-c399-4ffb-8320-8f18588385ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:48.534799023Z\",\n \"endTime\": \"2026-09-20T18:18:50.467299699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a6907afe93b489df04a82546f5045cc945e5154bbf8ca037a7c533d981852c83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"f37d1bd86a2dec4c46be9ce8fb4c03ed08425d29a0d32e20dd962b464a680228","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000696760\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-lzkpz5ftlkll\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\",\n \"name\": \"83245618-9845-4872-97a3-4ee524027d39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:02.101717721Z\",\n \"endTime\": \"2026-09-20T18:19:10.391207338Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000696760\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-lzkpz5ftlkll\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\",\n \"name\": \"83245618-9845-4872-97a3-4ee524027d39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:02.101717721Z\",\n \"endTime\": \"2026-09-20T18:19:10.391207338Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"44fc5c170ee5969da686dbaeb61a39c91b4e9f01cf7a4324b16151d7f8a7811a","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b688759f4d2ab67c229ce65b6ddfb3031692a5c8bd4c2a7870e657c2cb70a837","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"e2e","signature_id":"7486390772235b7b4ad872f03600862b7205a72c98df5cd28e716390fedf267b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-gns5q5 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ec160\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-gns5q5: failed waiting for cluster=\"np-version-upgrade-cluster-gns5q5\" in resourcegroup=\"rg-np-version-upgrade-gns5q5-hq6mhxkh7bdl\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4673ce57-b0e5-4848-88c0-86c48e4b213b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4673ce57-b0e5-4848-88c0-86c48e4b213b\",\n \"name\": \"4673ce57-b0e5-4848-88c0-86c48e4b213b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:48.158843811Z\",\n \"endTime\": \"2026-09-20T18:18:50.501392807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-gns5q5 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ec160\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-gns5q5: failed waiting for cluster=\"np-version-upgrade-cluster-gns5q5\" in resourcegroup=\"rg-np-version-upgrade-gns5q5-hq6mhxkh7bdl\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4673ce57-b0e5-4848-88c0-86c48e4b213b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4673ce57-b0e5-4848-88c0-86c48e4b213b\",\n \"name\": \"4673ce57-b0e5-4848-88c0-86c48e4b213b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:49:48.158843811Z\",\n \"endTime\": \"2026-09-20T18:18:50.501392807Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c057079edbc6dcf512a4ce5da52ef44c89587a9d7146e3535615d6a7b95ab7c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","failed_at":"alert","signature_id":"8389beb2f2127c28d1c407bb2631de05104a4b0dfdbf0a89c145c62d14049937","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 16 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-20T18:24:37Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-20T18:24:37Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2susts60dnkbabh5ft9bj4gfhtr5ubsk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2susts60dnkbabh5ft9bj4gfhtr5ubsk/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-20T18:26:36Z\n Ended: 2026-09-20T18:32:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2susu70nvmv9oua374lkat5p6me0e8uj\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2susu70nvmv9oua374lkat5p6me0e8uj/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","44fc5c170ee5969da686dbaeb61a39c91b4e9f01cf7a4324b16151d7f8a7811a","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101718617941872640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T17:02:21Z","started_at":"2026-09-20T17:02:21Z","completed_at":"2026-09-20T17:39:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T17:48:53.075215375Z","timing_metadata_checked_at":"2026-09-20T17:48:53.075215375Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T17:48:53.079094461Z","region_metadata_checked_at":"2026-09-20T17:48:53.079094461Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c586a78806d92ee0c5fb78f36f018eadb2ffc0c4235cb63fb6c858a453e907eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101718617941872640","occurred_at":"2026-09-20T17:02:21Z","failed_at":"provision","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-20T17:31:48.163Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-20T17:31:48.670Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-20T17:31:49.380Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1872640-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6\ntime=2026-09-20T17:33:49.754Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.Resources/deployments/113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6/operationStatuses/08584116813764412291\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-post\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"NotFound\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e07cda02c84cd6ea2552fbcc5179ebfd519e66665b62547f1ea88bc8136bc936","failure_pattern":"ERROR CODE: DeploymentFailed; detail code NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e07cda02c84cd6ea2552fbcc5179ebfd519e66665b62547f1ea88bc8136bc936"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101715422586867712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7033,"pr_state":"closed","pr_sha":"7dc06a6c4270438febda85df95c9bddffa9eb2c6","final_merged_sha":"ef821496d20065a3db001282a41d83827d994051","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T16:49:39Z","started_at":"2026-09-20T16:49:39Z","completed_at":"2026-09-20T19:06:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T19:12:48.083790668Z","timing_metadata_checked_at":"2026-09-20T19:12:48.083790668Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T19:12:48.090248749Z","region_metadata_checked_at":"2026-09-20T19:12:48.090248749Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"17a14a0957a4b84810803dbeea78325f1fe2cfd61a0d9d244ccc1397a6e55099","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101715422586867712","occurred_at":"2026-09-20T16:49:39Z","failed_at":"alert","signature_id":"a1c099010ae016739013a1e87affb32949a8f1160e4fb45e709b3366b93a0cff","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T18:17:23Z\n Ended: 2026-09-20T18:33:23Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6867712-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j6867712-svc).\nFiring 2:\n State: Resolved\n Started: 2026-09-20T18:17:23Z\n Ended: 2026-09-20T18:32:24Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6867712-svc\", component=\"frontend\", method=\"delete\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j6867712-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"66114193321438d473fff92dbe41e69829ce45a7dd02d3c9116ac30b8d4e7ab2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101715422586867712","occurred_at":"2026-09-20T16:49:39Z","failed_at":"e2e","signature_id":"c01cceb55dbabd6acf2c61e2aecc8f6bbf9895f9ca156a63db239382788d9d1e","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000eb2060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-8mmkqkc7dspl\" to finish deleting: GET https://rp.j6867712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f73e80f-46b4-48bf-9496-30e3a271cd59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f73e80f-46b4-48bf-9496-30e3a271cd59\",\n \"name\": \"9f73e80f-46b4-48bf-9496-30e3a271cd59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T18:26:50.030271411Z\",\n \"endTime\": \"2026-09-20T18:50:50.442666544Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suspvnhq9bs89art5mn22rcv6o4gr53 still exists (deletion dispatched at 2026-09-20T18:26:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000eb2060\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-8mmkqkc7dspl\" to finish deleting: GET https://rp.j6867712.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f73e80f-46b4-48bf-9496-30e3a271cd59\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9f73e80f-46b4-48bf-9496-30e3a271cd59\",\n \"name\": \"9f73e80f-46b4-48bf-9496-30e3a271cd59\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T18:26:50.030271411Z\",\n \"endTime\": \"2026-09-20T18:50:50.442666544Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suspvnhq9bs89art5mn22rcv6o4gr53 still exists (deletion dispatched at 2026-09-20T18:26:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101708991846420480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7032,"pr_state":"closed","pr_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","final_merged_sha":"c17bf1b8be7e620f98afeeb00dde43c86e8ebf20","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T16:24:06Z","started_at":"2026-09-20T16:24:06Z","completed_at":"2026-09-20T18:29:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T18:41:06.241339209Z","timing_metadata_checked_at":"2026-09-20T18:41:06.241339209Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T18:41:06.245039896Z","region_metadata_checked_at":"2026-09-20T18:41:06.245039896Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"04189d0f5e20a17650ae09a53aee20f93e067674440838af21ebe94baf5efb16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101708991846420480","occurred_at":"2026-09-20T16:24:06Z","failed_at":"e2e","signature_id":"7a7b14873ebc7e928a8df3faa64a26b8a0c2ace4736d3d613f6a89c5b2c7ebba","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00116c378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-9t8q6g\" in resourcegroup=\"rg-np-version-upgrade-9t8q6g-d8vnpqdkzzdd\" to finish deleting: GET https://rp.j6420480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\",\n \"name\": \"a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:41:51.558513909Z\",\n \"endTime\": \"2026-09-20T18:05:54.797697774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus7kr9oc70h1qkt3fma8jukokvb4fq still exists (deletion dispatched at 2026-09-20T17:41:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00116c378\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-9t8q6g\" in resourcegroup=\"rg-np-version-upgrade-9t8q6g-d8vnpqdkzzdd\" to finish deleting: GET https://rp.j6420480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\",\n \"name\": \"a221b1ff-c2d4-4d3e-bb9c-9102c33dcfa6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:41:51.558513909Z\",\n \"endTime\": \"2026-09-20T18:05:54.797697774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus7kr9oc70h1qkt3fma8jukokvb4fq still exists (deletion dispatched at 2026-09-20T17:41:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ba7095bc9215bcd43f1ce1c9be66b542a2bb9543fff4f416ae7f7cd986a2f3b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101708991846420480","occurred_at":"2026-09-20T16:24:06Z","failed_at":"e2e","signature_id":"f680e9c6bf353ff7e3737b28ae5ee7746498d1917182a5b8a67f475e211714a3","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012d8150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-77nnw8xhpg6s\" to finish deleting: GET https://rp.j6420480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83c11edb-cb7a-458f-8d8e-0fa679b34fd8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83c11edb-cb7a-458f-8d8e-0fa679b34fd8\",\n \"name\": \"83c11edb-cb7a-458f-8d8e-0fa679b34fd8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:48:44.753307034Z\",\n \"endTime\": \"2026-09-20T18:12:50.043857017Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus7ldf7cgmduvr2ibfvtv6auch4u6v still exists (deletion dispatched at 2026-09-20T17:48:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012d8150\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-77nnw8xhpg6s\" to finish deleting: GET https://rp.j6420480.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83c11edb-cb7a-458f-8d8e-0fa679b34fd8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83c11edb-cb7a-458f-8d8e-0fa679b34fd8\",\n \"name\": \"83c11edb-cb7a-458f-8d8e-0fa679b34fd8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:48:44.753307034Z\",\n \"endTime\": \"2026-09-20T18:12:50.043857017Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus7ldf7cgmduvr2ibfvtv6auch4u6v still exists (deletion dispatched at 2026-09-20T17:48:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7030,"pr_state":"closed","pr_sha":"4dc2c443d9d2f9fa2a598585253f8231c7ddf46c","final_merged_sha":"50f9710d6eb949ad5cf7c325f7c4d6cce9083ddf","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T16:10:57Z","started_at":"2026-09-20T16:10:57Z","completed_at":"2026-09-20T18:22:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T18:30:39.734701475Z","timing_metadata_checked_at":"2026-09-20T18:30:39.734701475Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T18:30:39.738181764Z","region_metadata_checked_at":"2026-09-20T18:30:39.738181764Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"510d5decb7dbc656e3bab754252f21edb285f50b483628007149b22b8f1c8352","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T17:13:02Z\n Ended: 2026-09-20T17:29:03Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5465216-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"6b7d8fb73aaeedccd83485a3889248f07a8bebc749a873d77b78ee01c97cf23b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","failed_at":"e2e","signature_id":"b369fa1435784bfd120c7576bed94a6e4a39fcf725052260c677eef0e62863e5","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010ca0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-v62fgp56rkgl\" to finish deleting: GET https://rp.j5465216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00507be0-8c9c-4d23-b9ba-c8c11bbcd318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00507be0-8c9c-4d23-b9ba-c8c11bbcd318\",\n \"name\": \"00507be0-8c9c-4d23-b9ba-c8c11bbcd318\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:32:28.39845259Z\",\n \"endTime\": \"2026-09-20T17:56:32.271542783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus5u9v5u2tuv9eqg0al8679ob1cvtj still exists (deletion dispatched at 2026-09-20T17:32:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010ca0a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-v62fgp56rkgl\" to finish deleting: GET https://rp.j5465216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00507be0-8c9c-4d23-b9ba-c8c11bbcd318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00507be0-8c9c-4d23-b9ba-c8c11bbcd318\",\n \"name\": \"00507be0-8c9c-4d23-b9ba-c8c11bbcd318\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:32:28.39845259Z\",\n \"endTime\": \"2026-09-20T17:56:32.271542783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus5u9v5u2tuv9eqg0al8679ob1cvtj still exists (deletion dispatched at 2026-09-20T17:32:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a789f4ea74f6d398875de4e36ef0c56467e2d8149e7a77e09db5818d8f340316","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","failed_at":"e2e","signature_id":"2eff4fde8ffd9f3247fb66660611f250c208eb3abb98228934a36763af0a55d7","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000c3c360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-s96znfczgxhl\" to finish deleting: GET https://rp.j5465216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b6ea12d-1868-4480-9201-fbbeb535fe47\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b6ea12d-1868-4480-9201-fbbeb535fe47\",\n \"name\": \"1b6ea12d-1868-4480-9201-fbbeb535fe47\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:40:48.648177125Z\",\n \"endTime\": \"2026-09-20T18:04:52.302253283Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus4q0ga6s9rpgj08rupc4dvqq0lr7j still exists (deletion dispatched at 2026-09-20T17:40:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000c3c360\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-s96znfczgxhl\" to finish deleting: GET https://rp.j5465216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b6ea12d-1868-4480-9201-fbbeb535fe47\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b6ea12d-1868-4480-9201-fbbeb535fe47\",\n \"name\": \"1b6ea12d-1868-4480-9201-fbbeb535fe47\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:40:48.648177125Z\",\n \"endTime\": \"2026-09-20T18:04:52.302253283Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sus4q0ga6s9rpgj08rupc4dvqq0lr7j still exists (deletion dispatched at 2026-09-20T17:40:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7029/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101687697306292224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7029,"pr_state":"closed","pr_sha":"0ddd8cc7d917131bb10132dfa6c0291c472c1b5d","final_merged_sha":"0ddd8cc7d917131bb10132dfa6c0291c472c1b5d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T14:59:29Z","started_at":"2026-09-20T14:59:29Z","completed_at":"2026-09-20T16:54:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T17:06:49.068278175Z","timing_metadata_checked_at":"2026-09-20T17:06:49.068278175Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T17:06:49.072180963Z","region_metadata_checked_at":"2026-09-20T17:06:49.072180963Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101683713959006208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T14:43:39Z","started_at":"2026-09-20T14:43:39Z","completed_at":"2026-09-20T17:08:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T17:17:11.794656569Z","timing_metadata_checked_at":"2026-09-20T17:17:11.794656569Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T17:17:11.798326057Z","region_metadata_checked_at":"2026-09-20T17:17:11.798326057Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"41ea2612881829bb7dc4582ecca40194a0f67e5781273ef7f1b3bad345f8aa3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101683713959006208","occurred_at":"2026-09-20T14:43:39Z","failed_at":"alert","signature_id":"19e557aaac4e676680853d6de094c41a779d1dca118ed39692cb9878dfd677f7","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T15:44:51Z\n Ended: 2026-09-20T16:04:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9006208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2suqt99oepq62cp9g8imp789kb8cucv2-u7z6v1b8p9z3l2n\", pod=\"router-75ddb8448b-mcjrz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2suqt99oepq62cp9g8imp789kb8cucv2-u7z6v1b8p9z3l2n/router-75ddb8448b-mcjrz has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-20T15:45:49Z\n Ended: 2026-09-20T16:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9006208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2suqteqhtuf74ce2qrveg0u8hgjr0p5r-e7c9f2r2o1e9v6w\", pod=\"router-7d46777b99-llb2x\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2suqteqhtuf74ce2qrveg0u8hgjr0p5r-e7c9f2r2o1e9v6w/router-7d46777b99-llb2x has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-20T15:45:49Z\n Ended: 2026-09-20T16:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9006208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2suqudsrpaqa5rkctesukjsg27o2l1a8-j6b1k8a5z0w9b4r\", pod=\"router-68967d6b9b-mjwwc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2suqudsrpaqa5rkctesukjsg27o2l1a8-j6b1k8a5z0w9b4r/router-68967d6b9b-mjwwc has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-20T15:46:51Z\n Ended: 2026-09-20T16:04:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9006208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2suqtr0pu725fj3uus9447oj3b8r0ga6-d9e8m6u9f5g8f7e\", pod=\"router-bf89df67f-dc5rn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2suqtr0pu725fj3uus9447oj3b8r0ga6-d9e8m6u9f5g8f7e/router-bf89df67f-dc5rn has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c972a5a828f0470963a3b4b90d8a4fd80eb05f65468d3843254d0b563dbaa2af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101683713959006208","occurred_at":"2026-09-20T14:43:39Z","failed_at":"alert","signature_id":"fb812e0e230afdfaf46a5827aae975ba58d618c82618d4f8ed79f5468297e6a0","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-20T16:39:02Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9006208-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vbl5w\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-vbl5w (namespace arobit) on cluster ci01-j9006208-svc has averaged 126.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T14:02:20Z","started_at":"2026-09-20T14:02:20Z","completed_at":"2026-09-20T15:57:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T16:03:53.495620959Z","timing_metadata_checked_at":"2026-09-20T16:03:53.495620959Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T16:03:53.499134818Z","region_metadata_checked_at":"2026-09-20T16:03:53.499134818Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"41c32a3a2824f046723c236a74b45faee8f8ebd5459b2096613a68a4dcbdbfbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","failed_at":"e2e","signature_id":"21b7ce7b54d77195fc617e821329b90e750c10192c6be2fdd3f4089b6662713d","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cbe240\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mhtls7w2h6hh\" to finish creating: GET https://rp.j6358528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1160ee4-c1b1-4f87-a844-83d0551e46a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1160ee4-c1b1-4f87-a844-83d0551e46a1\",\n \"name\": \"e1160ee4-c1b1-4f87-a844-83d0551e46a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T14:49:03.140717865Z\",\n \"endTime\": \"2026-09-20T15:18:05.481184474Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cbe240\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mhtls7w2h6hh\" to finish creating: GET https://rp.j6358528.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1160ee4-c1b1-4f87-a844-83d0551e46a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1160ee4-c1b1-4f87-a844-83d0551e46a1\",\n \"name\": \"e1160ee4-c1b1-4f87-a844-83d0551e46a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T14:49:03.140717865Z\",\n \"endTime\": \"2026-09-20T15:18:05.481184474Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d799d1f5a1cc22da71d3398ecf519fc25a4c93ad679c4139fdfc45d659168f70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","failed_at":"alert","signature_id":"e66e5d2206cbbff91631193f79b282c5f04f3b6efea313eb1c9fc43af3b6f8ad","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T15:11:10Z\n Ended: 2026-09-20T15:36:05Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j6358528-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.107:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6358528/providers/microsoft.monitor/accounts/hcps-j6358528\", namespace=\"ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-x5q5k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"e23140e3f627c654ababf7eba526daf0cc71c26b8e8cee26d7b7ae12c1588d9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","failed_at":"alert","signature_id":"84036f94843f01e49f13138f55d77ae74e49b96060ac2cc402fa01f03e462128","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T15:26:04Z\n Ended: 2026-09-20T15:41:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6358528-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.107:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6358528/providers/microsoft.monitor/accounts/hcps-j6358528\", namespace=\"ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-x5q5k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673266119708672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7025,"pr_state":"closed","pr_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","final_merged_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T14:02:08Z","started_at":"2026-09-20T14:02:08Z","completed_at":"2026-09-20T15:59:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T16:14:44.390494098Z","timing_metadata_checked_at":"2026-09-20T16:14:44.390494098Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T16:14:44.393971187Z","region_metadata_checked_at":"2026-09-20T16:14:44.393971187Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101670587393904640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7026,"pr_state":"closed","pr_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","final_merged_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T13:51:29Z","started_at":"2026-09-20T13:51:29Z","completed_at":"2026-09-20T16:01:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T16:14:44.493587366Z","timing_metadata_checked_at":"2026-09-20T16:14:44.493587366Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T16:14:44.496047758Z","region_metadata_checked_at":"2026-09-20T16:14:44.496047758Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f15241c0a1f51c5285e92def8ad6c40e26a0cfd063837b73ec858ce9a07a2fff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101670587393904640","occurred_at":"2026-09-20T13:51:29Z","failed_at":"e2e","signature_id":"10316e5a87c9726ae96606bd5b16607ddf7b0d400689c8e8c0e050115901ad56","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017da108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7v2dcntmwrc6\" to finish deleting: GET https://rp.j3904640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05e626d4-f861-4914-be3d-d9945e36c566\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05e626d4-f861-4914-be3d-d9945e36c566\",\n \"name\": \"05e626d4-f861-4914-be3d-d9945e36c566\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T15:22:08.248375345Z\",\n \"endTime\": \"2026-09-20T15:46:08.421611372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suq3e6ic386bksmrgs696k8ca3rriju still exists (deletion dispatched at 2026-09-20T15:22:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017da108\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-7v2dcntmwrc6\" to finish deleting: GET https://rp.j3904640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05e626d4-f861-4914-be3d-d9945e36c566\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/05e626d4-f861-4914-be3d-d9945e36c566\",\n \"name\": \"05e626d4-f861-4914-be3d-d9945e36c566\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T15:22:08.248375345Z\",\n \"endTime\": \"2026-09-20T15:46:08.421611372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suq3e6ic386bksmrgs696k8ca3rriju still exists (deletion dispatched at 2026-09-20T15:22:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101647291294158848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7025,"pr_state":"closed","pr_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","final_merged_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T12:18:55Z","started_at":"2026-09-20T12:18:55Z","completed_at":"2026-09-20T13:57:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T14:07:52.394475216Z","timing_metadata_checked_at":"2026-09-20T14:07:52.394475216Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T14:07:52.397835104Z","region_metadata_checked_at":"2026-09-20T14:07:52.397835104Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5d11569701ec4fedd520cab68d1579802c7157ba87b8355b91ce4c44dc6f1a35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101647291294158848","occurred_at":"2026-09-20T12:18:55Z","failed_at":"e2e","signature_id":"e0ec91f3cf6a62bae5b4c3d56509ed71730ec1128a9898d0313dd55a382d4fa9","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:184]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ca8a20\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:184]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ca8a20\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","failure_pattern":"UpdateHCPCluster20260901 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643653972758528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7030,"pr_state":"closed","pr_sha":"4dc2c443d9d2f9fa2a598585253f8231c7ddf46c","final_merged_sha":"50f9710d6eb949ad5cf7c325f7c4d6cce9083ddf","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-20T12:04:28Z","started_at":"2026-09-20T12:04:28Z","completed_at":"2026-09-20T14:06:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T14:18:22.536443847Z","timing_metadata_checked_at":"2026-09-20T14:18:22.536443847Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T14:18:22.539587236Z","region_metadata_checked_at":"2026-09-20T14:18:22.539587236Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"34e9c57ed0be8aa68a4d9c94a38d30de4f2709abb5ea2f95778c521dc5c58c4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643653972758528","occurred_at":"2026-09-20T12:04:28Z","failed_at":"alert","signature_id":"f483622241db14070571766c5ea34d86fd6f53d96057f1964f32d162917e2451","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T13:27:17Z\n Ended: 2026-09-20T13:41:19Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j2758528-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2758528-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7029/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643457066962944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7029,"pr_state":"closed","pr_sha":"0ddd8cc7d917131bb10132dfa6c0291c472c1b5d","final_merged_sha":"0ddd8cc7d917131bb10132dfa6c0291c472c1b5d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T12:03:41Z","started_at":"2026-09-20T12:03:41Z","completed_at":"2026-09-20T14:21:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T14:28:56.140128041Z","timing_metadata_checked_at":"2026-09-20T14:28:56.140128041Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T14:28:56.14324203Z","region_metadata_checked_at":"2026-09-20T14:28:56.14324203Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"45895f26687a3e4c5bbb69605b2f68bdd724847a15f09cbd57f0cfc433e842e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7029/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643457066962944","occurred_at":"2026-09-20T12:03:41Z","failed_at":"e2e","signature_id":"5fb2a2918cf998e33798a60ea71579a7a8d626d7c294f1e7d5559e93c8cb6517","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c380d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xw2mr9c7sf7t\" to finish deleting: GET https://rp.j6962944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13baf428-ed80-4931-a4b3-8398b975e18c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13baf428-ed80-4931-a4b3-8398b975e18c\",\n \"name\": \"13baf428-ed80-4931-a4b3-8398b975e18c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T13:41:50.034520019Z\",\n \"endTime\": \"2026-09-20T14:05:52.42378958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suoj1308cgeas4uvag9eqertadh2q3h still exists (deletion dispatched at 2026-09-20T13:41:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c380d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-xw2mr9c7sf7t\" to finish deleting: GET https://rp.j6962944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13baf428-ed80-4931-a4b3-8398b975e18c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13baf428-ed80-4931-a4b3-8398b975e18c\",\n \"name\": \"13baf428-ed80-4931-a4b3-8398b975e18c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T13:41:50.034520019Z\",\n \"endTime\": \"2026-09-20T14:05:52.42378958Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2suoj1308cgeas4uvag9eqertadh2q3h still exists (deletion dispatched at 2026-09-20T13:41:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T12:02:36Z","started_at":"2026-09-20T12:02:36Z","completed_at":"2026-09-20T13:40:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T13:46:25.016731887Z","timing_metadata_checked_at":"2026-09-20T13:46:25.016731887Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T13:46:25.019790678Z","region_metadata_checked_at":"2026-09-20T13:46:25.019790678Z"},"failed_at":["e2e"],"failed_test_count":21,"occurrences":[{"row_id":"29ce18dea7d175c68db3a3ef5f84e9381c6919f529a5085f6de3dd2209e22beb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"91a719e5ea3aea8b9b0e188fedf3ccfcff48a4a62dc5fc1afe26e222804d6227","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-5-0-7gj726\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001893500\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-zstream-5-0-7gj726-5thxzf\" in resourcegroup=\"rg-zstream-upgrade-5-0-nvfh99dzvbhb\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-5-0-nvfh99dzvbhb/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-5-0-7gj726-5thxzf/operationStatuses/08584116992455805325\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:106]: failed to create customer resources for z-stream cluster \"cluster-zstream-5-0-7gj726\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001893500\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-zstream-5-0-7gj726-5thxzf\" in resourcegroup=\"rg-zstream-upgrade-5-0-nvfh99dzvbhb\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-zstream-upgrade-5-0-nvfh99dzvbhb/providers/Microsoft.Resources/deployments/customer-infra-cluster-zstream-5-0-7gj726-5thxzf/operationStatuses/08584116992455805325\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"34810f1b5359320d18ca47e793c3086eaaf20eb9e286ce83d65e9dd26aa6edf2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"75b9233e5419e103c738cc032e9a0d4380e13a4936bd5bad2e51413dae0bf26f","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002b80c0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-arm64-vm-hcp-cluster-q2fmvf\" in resourcegroup=\"arm64-vm-cluster-6ct9s56bgsnd\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-6ct9s56bgsnd/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-q2fmvf/operationStatuses/08584116992449832602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002b80c0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-arm64-vm-hcp-cluster-q2fmvf\" in resourcegroup=\"arm64-vm-cluster-6ct9s56bgsnd\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-6ct9s56bgsnd/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-q2fmvf/operationStatuses/08584116992449832602\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"3d854a444148a1fe3d93332ca05ff812f13dce922681796f90959e3bc583aa07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"9db4a31a545e08b96a9f9b2a51e0ec6fa079b7393b1270544cac4af6f6246fd5","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a714e0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-idms-e2e-hcp-cluster-nh4pbh\" in resourcegroup=\"idms-l5zltlxr72bw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/idms-l5zltlxr72bw/providers/Microsoft.Resources/deployments/customer-infra-idms-e2e-hcp-cluster-nh4pbh/operationStatuses/08584116992375018318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:90]: failed to create customer resources for IDMS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a714e0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-idms-e2e-hcp-cluster-nh4pbh\" in resourcegroup=\"idms-l5zltlxr72bw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/idms-l5zltlxr72bw/providers/Microsoft.Resources/deployments/customer-infra-idms-e2e-hcp-cluster-nh4pbh/operationStatuses/08584116992375018318\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"5ae5159e14bf52b4e12cef329e6c3c98b41e6c39ed772fc447c1107dfa2c619a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"b9513cbbafa37244dbcd3207b8b1836ea824ef6ed8a8fa6ee5005da287c245c1","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015224a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-dg-minor-97cpdn-97nfmz\" in resourcegroup=\"rg-np-dg-minor-97cpdn-jwdnmqz4h5k5\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-dg-minor-97cpdn-jwdnmqz4h5k5/providers/Microsoft.Resources/deployments/customer-infra-np-dg-minor-97cpdn-97nfmz/operationStatuses/08584116992296089268\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:710]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015224a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-dg-minor-97cpdn-97nfmz\" in resourcegroup=\"rg-np-dg-minor-97cpdn-jwdnmqz4h5k5\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-dg-minor-97cpdn-jwdnmqz4h5k5/providers/Microsoft.Resources/deployments/customer-infra-np-dg-minor-97cpdn-97nfmz/operationStatuses/08584116992296089268\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"68446b8a5e2ce286646f100ed7f22e624a7e85a5a990e79c0e247c0429e70fe5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"bd221a69442a6d79a216e48307db3abafc2724516fd0d3101335d3d4bb683a2d","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0aa80\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-version-upgrade-cluster-5sgmvt-wh2fh7\" in resourcegroup=\"rg-np-version-upgrade-5sgmvt-ddwpmwwlkbp8\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-5sgmvt-ddwpmwwlkbp8/providers/Microsoft.Resources/deployments/customer-infra-np-version-upgrade-cluster-5sgmvt-wh2fh7/operationStatuses/08584116992292355490\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0aa80\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-version-upgrade-cluster-5sgmvt-wh2fh7\" in resourcegroup=\"rg-np-version-upgrade-5sgmvt-ddwpmwwlkbp8\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-5sgmvt-ddwpmwwlkbp8/providers/Microsoft.Resources/deployments/customer-infra-np-version-upgrade-cluster-5sgmvt-wh2fh7/operationStatuses/08584116992292355490\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6a78de9e602b3241a40519368cf2bd66924ae7f395ee9425783080a028598d67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"1a79c558038cb73ff72359e020fc0d6f9f740e31cbd0af1daebaf91ad6c65fa3","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:76]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f02d60\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kas-m75gws\" in resourcegroup=\"private-kas-s98lp67jv589\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-kas-s98lp67jv589/providers/Microsoft.Resources/deployments/customer-infra-private-kas-m75gws/operationStatuses/08584116992457285882\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:76]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f02d60\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kas-m75gws\" in resourcegroup=\"private-kas-s98lp67jv589\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-kas-s98lp67jv589/providers/Microsoft.Resources/deployments/customer-infra-private-kas-m75gws/operationStatuses/08584116992457285882\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6bd3fc1c3ef6e8fda466478243f384ca27bd1743540d8d7b525c69212e27b9e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"454410db4b0c9c7e89e9e307d8cabd3be3a5cc6c620a919683872059242695a3","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:72]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb0540\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-fips-enabled-cluster-d9t886\" in resourcegroup=\"fips-enabled-rvgn6tk7vl7q\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/fips-enabled-rvgn6tk7vl7q/providers/Microsoft.Resources/deployments/customer-infra-fips-enabled-cluster-d9t886/operationStatuses/08584116992455136747\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:72]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb0540\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-fips-enabled-cluster-d9t886\" in resourcegroup=\"fips-enabled-rvgn6tk7vl7q\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/fips-enabled-rvgn6tk7vl7q/providers/Microsoft.Resources/deployments/customer-infra-fips-enabled-cluster-d9t886/operationStatuses/08584116992455136747\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6c240beaa637653d1dfc84c9fe493b860a14ef71913a42d125539a3d36adb8c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"56478c58658652b4bc1945a590ac05ad32b42a6f22c2e138039cb7ee3c4cbfac","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2f940\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-7mb4tc\" in resourcegroup=\"external-auth-cluster-bhfp8rbccvvf\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-bhfp8rbccvvf/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-7mb4tc/operationStatuses/08584116992371740355\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:85]: failed to create customer resources for external auth cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2f940\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-7mb4tc\" in resourcegroup=\"external-auth-cluster-bhfp8rbccvvf\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-bhfp8rbccvvf/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-7mb4tc/operationStatuses/08584116992371740355\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"6cbb32f2d070654308a2db71c06b3791edd01d3792fdd3c8f1af09b7fa8b4be7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"a186884d8b4f85878973e1e8bb26766fcd36f0f99234916982d31c2386197c2a","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:74]: failed to create customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d89980\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-pause-bkp-cluster-9zrvd8\" in resourcegroup=\"pause-bkp-e2e-5kfj7kdvg7fr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/pause-bkp-e2e-5kfj7kdvg7fr/providers/Microsoft.Resources/deployments/customer-infra-pause-bkp-cluster-9zrvd8/operationStatuses/08584116992375377229\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:74]: failed to create customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d89980\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-pause-bkp-cluster-9zrvd8\" in resourcegroup=\"pause-bkp-e2e-5kfj7kdvg7fr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/pause-bkp-e2e-5kfj7kdvg7fr/providers/Microsoft.Resources/deployments/customer-infra-pause-bkp-cluster-9zrvd8/operationStatuses/08584116992375377229\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7a9b88fe228b0082f576b5fb455ed426081893406605762f234f792888ba1a66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"42440a2d7bb764d5edcdcdcd65e32a762540470eaa9f7ce1559f0c2a375c81a0","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b32200\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-autoscale-cluster-gz6j7m\" in resourcegroup=\"np-autoscaling-d6nt8sqhc4v2\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-d6nt8sqhc4v2/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-gz6j7m/operationStatuses/08584116992287759958\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:93]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b32200\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-autoscale-cluster-gz6j7m\" in resourcegroup=\"np-autoscaling-d6nt8sqhc4v2\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-d6nt8sqhc4v2/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-gz6j7m/operationStatuses/08584116992287759958\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"7ba761628929a594d4d994b456f8ce091cb1625060cc24b8405a3a1a80d31e25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"0899e170df3dbff91a3d93d67607d26e9d9e1d55e607106e4712f92b087c03fd","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0027fddb0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0027fddb0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}},{"row_id":"807a4aa3e07746639902538cdb3c7f789ffe3f005cda5c48a49cb81c2f622ac3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"81b579506962c7694460b4e0ba3269306f93397ebe18eb85545cae4066854205","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001782300\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-version-upgrade-cluster-x7b8c9-g9hng6\" in resourcegroup=\"rg-np-version-upgrade-x7b8c9-78csh8btgppk\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-x7b8c9-78csh8btgppk/providers/Microsoft.Resources/deployments/customer-infra-np-version-upgrade-cluster-x7b8c9-g9hng6/operationStatuses/08584116992297707311\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001782300\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-np-version-upgrade-cluster-x7b8c9-g9hng6\" in resourcegroup=\"rg-np-version-upgrade-x7b8c9-78csh8btgppk\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-x7b8c9-78csh8btgppk/providers/Microsoft.Resources/deployments/customer-infra-np-version-upgrade-cluster-x7b8c9-g9hng6/operationStatuses/08584116992297707311\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"87bfff40d4a206f858615acd48a139316c21612c4b6785262b6be28bb3d66030","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"d798f3f3a8a081a020bf7521957c622331f9901d07425be35f0617500d755d67","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-8n7lrz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00096f0a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-22-8n7lrz-jq5k6h\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v9fgb6qclpxh\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-22-v9fgb6qclpxh/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-22-8n7lrz-jq5k6h/operationStatuses/08584116992380864708\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-22-8n7lrz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00096f0a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-22-8n7lrz-jq5k6h\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v9fgb6qclpxh\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-22-v9fgb6qclpxh/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-22-8n7lrz-jq5k6h/operationStatuses/08584116992380864708\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"97e5e7db391a5ea3a14515531b6e53e2f94b88bf52da9410a7e2c61c102543d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"73c41b364dee2fd3eb27b4ccaa57fb601deed774cfcc5111654a86e31aa1c0ef","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:83]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fce560\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cilium-cluster-779vpn\" in resourcegroup=\"complex-cilium-kv-hj2fprxw8lj6\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/complex-cilium-kv-hj2fprxw8lj6/providers/Microsoft.Resources/deployments/customer-infra-cilium-cluster-779vpn/operationStatuses/08584116992458344159\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:83]: failed to create customer resources with private key vault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fce560\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cilium-cluster-779vpn\" in resourcegroup=\"complex-cilium-kv-hj2fprxw8lj6\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/complex-cilium-kv-hj2fprxw8lj6/providers/Microsoft.Resources/deployments/customer-infra-cilium-cluster-779vpn/operationStatuses/08584116992458344159\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"a6086b66362884bb02fb90142aa5fa84d59bd590d924f79597287be439d5e48b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"5d0ca65ae867bd15279982b3a25e60bccfc658f662dfee162bb114f2ddaccb2c","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98280\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-list6zv9h4-ncgnxk\" in resourcegroup=\"ea-list-4wzbljbzdfsj\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/ea-list-4wzbljbzdfsj/providers/Microsoft.Resources/deployments/customer-infra-ea-list6zv9h4-ncgnxk/operationStatuses/08584116992369169703\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:78]: failed to create customer resources for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a98280\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-list6zv9h4-ncgnxk\" in resourcegroup=\"ea-list-4wzbljbzdfsj\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/ea-list-4wzbljbzdfsj/providers/Microsoft.Resources/deployments/customer-infra-ea-list6zv9h4-ncgnxk/operationStatuses/08584116992369169703\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b3234b9726f3fc85edcde87feda0137418d489bd3afd0626bcb69689f1ea4472","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"a21cc835fa8cdb2ed9033f1ca18d1eb7db85fc60669d3f2f049f9f0f8e99c24d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:111]: failed to create customer resources for cluster \"cluster-candidate-4-20-kpzqjb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000793700\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-20-kpzqjb-rzndrn\" in resourcegroup=\"rg-candidate-4-20-hv7qvmmjkmk5\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-20-hv7qvmmjkmk5/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-20-kpzqjb-rzndrn/operationStatuses/08584116992454378325\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:111]: failed to create customer resources for cluster \"cluster-candidate-4-20-kpzqjb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000793700\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-20-kpzqjb-rzndrn\" in resourcegroup=\"rg-candidate-4-20-hv7qvmmjkmk5\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-20-hv7qvmmjkmk5/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-20-kpzqjb-rzndrn/operationStatuses/08584116992454378325\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c89daa424607cd7dee410ca9373d71e0c35ab6ae04c63c09dadb3673f0f3694f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"c3ed73d691c5929cdf2946a476c8b001c9536e6031f54f65e36b74ff7e76c90f","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:95]: failed to deploy customer infrastructure for back-level version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001009720\u003e: \n failed waiting for deployment \"customer-infra-cluster-ver-4-20-m6nnt5\" in resourcegroup=\"rg-cluster-back-version-gddcccf28p5t\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-back-version-gddcccf28p5t/providers/Microsoft.Resources/deployments/customer-infra-cluster-ver-4-20-m6nnt5/operationStatuses/08584116992459366081\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:95]: failed to deploy customer infrastructure for back-level version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001009720\u003e: \n failed waiting for deployment \"customer-infra-cluster-ver-4-20-m6nnt5\" in resourcegroup=\"rg-cluster-back-version-gddcccf28p5t\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-back-version-gddcccf28p5t/providers/Microsoft.Resources/deployments/customer-infra-cluster-ver-4-20-m6nnt5/operationStatuses/08584116992459366081\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"cc57aee78b3227375504cf7ced949360d61d6993c040319ea1cab2c1d1fb5e0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"2b974162c7e6d97b0de4aea836205f4f789664585153978c18ab76f65fd6ff59","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:78]: failed to create customer resources for KMS key rotation cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00091f420\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-kms-key-rotate-422-pxl46j\" in resourcegroup=\"kms-key-rotate-j9t2vs2r5nhz\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/kms-key-rotate-j9t2vs2r5nhz/providers/Microsoft.Resources/deployments/customer-infra-kms-key-rotate-422-pxl46j/operationStatuses/08584116992371388304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:78]: failed to create customer resources for KMS key rotation cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00091f420\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-kms-key-rotate-422-pxl46j\" in resourcegroup=\"kms-key-rotate-j9t2vs2r5nhz\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/kms-key-rotate-j9t2vs2r5nhz/providers/Microsoft.Resources/deployments/customer-infra-kms-key-rotate-422-pxl46j/operationStatuses/08584116992371388304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"da675a2341fc514db40cb885acb7d29eb2a6afa1d66a9f22d368a05ca6cdf8e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"1b003937044cfa4fcd5a2675364491c0f6ffb7db83c6398081a44e0f34d6bb6b","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a11f20\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-ltthln\" in resourcegroup=\"private-ingress-6mv7ddflq9wr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-6mv7ddflq9wr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-ltthln/operationStatuses/08584116992436980878\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a11f20\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-ltthln\" in resourcegroup=\"private-ingress-6mv7ddflq9wr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-6mv7ddflq9wr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-ltthln/operationStatuses/08584116992436980878\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"eaefbf5e1c83019d5b962dea3d56223c3e93de1b60d67adc79803f8228404ebb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"ab6420d788f7161106d32d50e735d0b55e0a7d0b7c8c03f40db9586c7ac5332f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-gn9sjc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000715ca0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-5-0-gn9sjc-kdgqtg\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-htm8g5gh72dw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-5-0-htm8g5gh72dw/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-5-0-gn9sjc-kdgqtg/operationStatuses/08584116992383393210\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-5-0-gn9sjc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000715ca0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-5-0-gn9sjc-kdgqtg\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-htm8g5gh72dw\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-5-0-htm8g5gh72dw/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-5-0-gn9sjc-kdgqtg/operationStatuses/08584116992383393210\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"f5f0967e26277b45e4b657784a4e7199419c1a1b3e1172b453c6c6ce989cbc2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","failed_at":"e2e","signature_id":"268106fe2eb57086c4eef48987e800efaac560af7778223cb42b3520a44d6749","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:110]: failed to create customer resources for cluster-subnet-reuse\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12720\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-h6k9tb\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-h6k9tb/operationStatuses/08584116992459228470\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:110]: failed to create customer resources for cluster-subnet-reuse\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12720\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-h6k9tb\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-h6k9tb/operationStatuses/08584116992459228470\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":21,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504"],"matched_occurrences":21,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101634420027953152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7026,"pr_state":"closed","pr_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","final_merged_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T11:27:46Z","started_at":"2026-09-20T11:27:46Z","completed_at":"2026-09-20T13:33:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T13:46:25.160762655Z","timing_metadata_checked_at":"2026-09-20T13:46:25.160762655Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T13:46:25.163101248Z","region_metadata_checked_at":"2026-09-20T13:46:25.163101248Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"01bf343ccafed062791df15003912369e2435fcca2f41ded1aabed6e980d7387","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101634420027953152","occurred_at":"2026-09-20T11:27:46Z","failed_at":"alert","signature_id":"5b8bff552d46579784f79a0d3b3120eddbccf7ef351d544b8715f930813682e3","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T12:53:00Z\n Ended: 2026-09-20T12:58:49Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j7953152-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j7953152-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"b71ddf1366183398b374e1610d716a27a88a350233cde6ca54a03ecea92f1717","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101634420027953152","occurred_at":"2026-09-20T11:27:46Z","failed_at":"e2e","signature_id":"b854729ec479cb89d9449d9a6d7d41e7ad976014ffc19decf951194f7d9ca05c","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a920f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-l2hrlk8rzgwm\" to finish deleting: GET https://rp.j7953152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5ce2f387-0594-454c-acf7-c24b6cbb1e1b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5ce2f387-0594-454c-acf7-c24b6cbb1e1b\",\n \"name\": \"5ce2f387-0594-454c-acf7-c24b6cbb1e1b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T12:54:10.213869417Z\",\n \"endTime\": \"2026-09-20T13:18:13.968831064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sunvib8s08n5gj9rrnedc5kn1khs4j8 still exists (deletion dispatched at 2026-09-20T12:54:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000a920f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-l2hrlk8rzgwm\" to finish deleting: GET https://rp.j7953152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5ce2f387-0594-454c-acf7-c24b6cbb1e1b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5ce2f387-0594-454c-acf7-c24b6cbb1e1b\",\n \"name\": \"5ce2f387-0594-454c-acf7-c24b6cbb1e1b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T12:54:10.213869417Z\",\n \"endTime\": \"2026-09-20T13:18:13.968831064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sunvib8s08n5gj9rrnedc5kn1khs4j8 still exists (deletion dispatched at 2026-09-20T12:54:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6941/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610749217476608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6941,"pr_state":"closed","pr_sha":"5196c129c2fe0fb3f9bfc329e0463402243206bc","final_merged_sha":"5196c129c2fe0fb3f9bfc329e0463402243206bc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T09:53:43Z","started_at":"2026-09-20T09:53:43Z","completed_at":"2026-09-20T12:00:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T12:11:30.615411636Z","timing_metadata_checked_at":"2026-09-20T12:11:30.615411636Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T12:11:30.618077928Z","region_metadata_checked_at":"2026-09-20T12:11:30.618077928Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T09:53:04Z","started_at":"2026-09-20T09:53:04Z","completed_at":"2026-09-20T11:57:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T12:00:33.598053486Z","timing_metadata_checked_at":"2026-09-20T12:00:33.598053486Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T12:00:33.600936977Z","region_metadata_checked_at":"2026-09-20T12:00:33.600936977Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"161289058ece2082e780c6021f6440e9c0b15d26d604d60ca9a1e195978f2214","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","failed_at":"e2e","signature_id":"8a93b4c61b0436ed275798352e60b97494e315997f9735dcbbf3fcea1cb8b1bc","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8d220\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-cc6rq5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-cc6rq5\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-pullsecret-hcp-cluster-cc6rq5' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8d220\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-cc6rq5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-cc6rq5\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-pullsecret-hcp-cluster-cc6rq5' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}},{"row_id":"a941b8a7843c480006f270a64d8a5b19b51b3a0e9acb730eb356c00083119535","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","failed_at":"alert","signature_id":"f6030bcc4e4cc5c82d977d09b32388275720da97127703ae213df38d33273f13","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T11:27:29Z\n Ended: 2026-09-20T11:33:32Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j2086400-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2086400-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"d85c78bee8d1d6813548e19fee771f2b404a0a4ce7351aa6ecf896ef948a1227","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","failed_at":"e2e","signature_id":"39cf09c4d4c3e066082fd6d0301325efef03f8489b16e7088e47c2ad9b88bbaa","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\nUnexpected error:\n \u003c*errors.StatusError | 0xc000ac2640\u003e: \n Operation cannot be fulfilled on nodes \"k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\nUnexpected error:\n \u003c*errors.StatusError | 0xc000ac2640\u003e: \n Operation cannot be fulfilled on nodes \"k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred"}},{"row_id":"e6cc162045a2e1ad5f7b6f7db2a97387484a19646003916e45dbacd874c354e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","failed_at":"e2e","signature_id":"7315612725ec25e205b11f3adbd85547f42e9e5cff4ae5b4c955333bbd4ccd4f","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-wg24nd-tjd5hx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-wg24nd-tjd5hx\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-np-version-upgrade-cluster-wg24nd-tjd5hx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-wg24nd-tjd5hx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-wg24nd-tjd5hx\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-np-version-upgrade-cluster-wg24nd-tjd5hx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610157367627776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7025,"pr_state":"closed","pr_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","final_merged_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T09:51:22Z","started_at":"2026-09-20T09:51:22Z","completed_at":"2026-09-20T12:02:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T12:11:31.002193402Z","timing_metadata_checked_at":"2026-09-20T12:11:31.002193402Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T12:11:31.004756094Z","region_metadata_checked_at":"2026-09-20T12:11:31.004756094Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b03c9a884adc647e77e1b18fa191a5eeeba6ce6deeee6752458ac91e9e041b3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610157367627776","occurred_at":"2026-09-20T09:51:22Z","failed_at":"e2e","signature_id":"d3228063fdb4db7077f94a323b61ddb40a6e50e0a6a4ce81d902c823eb177dc2","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001ba80f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-j5f5ss4sbkp7\" to finish deleting: GET https://rp.j7627776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/676ab8d4-2d39-4758-b787-1a8b8a8dad3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/676ab8d4-2d39-4758-b787-1a8b8a8dad3f\",\n \"name\": \"676ab8d4-2d39-4758-b787-1a8b8a8dad3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T11:23:08.876040701Z\",\n \"endTime\": \"2026-09-20T11:47:13.388195572Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sumjjp5k6vfqppk7421kh5jdhmndmk0 still exists (deletion dispatched at 2026-09-20T11:23:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001ba80f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-j5f5ss4sbkp7\" to finish deleting: GET https://rp.j7627776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/676ab8d4-2d39-4758-b787-1a8b8a8dad3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/676ab8d4-2d39-4758-b787-1a8b8a8dad3f\",\n \"name\": \"676ab8d4-2d39-4758-b787-1a8b8a8dad3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T11:23:08.876040701Z\",\n \"endTime\": \"2026-09-20T11:47:13.388195572Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sumjjp5k6vfqppk7421kh5jdhmndmk0 still exists (deletion dispatched at 2026-09-20T11:23:10Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101600364640866304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7026,"pr_state":"closed","pr_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","final_merged_sha":"e6f7a5a5e8aee3da7f3d48c65e2b9f726ece20d8","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T09:12:27Z","started_at":"2026-09-20T09:12:27Z","completed_at":"2026-09-20T11:21:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T11:28:32.587013286Z","timing_metadata_checked_at":"2026-09-20T11:28:32.587013286Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T11:28:32.589897477Z","region_metadata_checked_at":"2026-09-20T11:28:32.589897477Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"0f974f941334a44e14698348b15d174831988e5504e9e9d27fb043d8961369b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101600364640866304","occurred_at":"2026-09-20T09:12:27Z","failed_at":"alert","signature_id":"f3bf8d60286004f12d28b5f26936659fa6d1debd6a2443cbc6166170ce25bcb6","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T10:11:11Z\n Ended: 2026-09-20T10:45:10Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0866304-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0866304-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7023/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101588887028633600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7023,"pr_state":"closed","pr_sha":"6cae5a1171dd63e42d1171cdfc4ddd325d953fb9","final_merged_sha":"6cae5a1171dd63e42d1171cdfc4ddd325d953fb9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T08:26:50Z","started_at":"2026-09-20T08:26:50Z","completed_at":"2026-09-20T10:56:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T10:56:50.501476469Z","timing_metadata_checked_at":"2026-09-20T10:56:50.501476469Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T10:56:50.505736543Z","region_metadata_checked_at":"2026-09-20T10:56:50.505736543Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2b2b934509611a0e7e66acb3b999fe61265128d1009f064da68a6e1c2d973d61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7023/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101588887028633600","occurred_at":"2026-09-20T08:26:50Z","failed_at":"e2e","signature_id":"3a8359bef606bfcfc9ffd99f5dc07bf9b62993cb12fd5d570d46aa5683d99f29","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014a6168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-h2thhnsvkjqg\" to finish deleting: GET https://rp.j8633600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a58bfa81-14c0-418f-8038-eb180ea92a60\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a58bfa81-14c0-418f-8038-eb180ea92a60\",\n \"name\": \"a58bfa81-14c0-418f-8038-eb180ea92a60\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T10:14:25.908843484Z\",\n \"endTime\": \"2026-09-20T10:38:32.00607941Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sulilh97eig2ggpp4bpddu5dj29cgdm still exists (deletion dispatched at 2026-09-20T10:14:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014a6168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-h2thhnsvkjqg\" to finish deleting: GET https://rp.j8633600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a58bfa81-14c0-418f-8038-eb180ea92a60\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a58bfa81-14c0-418f-8038-eb180ea92a60\",\n \"name\": \"a58bfa81-14c0-418f-8038-eb180ea92a60\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T10:14:25.908843484Z\",\n \"endTime\": \"2026-09-20T10:38:32.00607941Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sulilh97eig2ggpp4bpddu5dj29cgdm still exists (deletion dispatched at 2026-09-20T10:14:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101575058240573440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7025,"pr_state":"closed","pr_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","final_merged_sha":"05239b6f7cd121f0e56e4ef67bd4ec223e1a5835","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T07:31:53Z","started_at":"2026-09-20T07:31:53Z","completed_at":"2026-09-20T09:24:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T09:33:14.856272513Z","timing_metadata_checked_at":"2026-09-20T09:33:14.856272513Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T09:33:14.86257549Z","region_metadata_checked_at":"2026-09-20T09:33:14.86257549Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f49a0483d2cfd6cf62acc18f050e8bfe7fd81a57d549cc1240ca7db758f041fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101575058240573440","occurred_at":"2026-09-20T07:31:53Z","failed_at":"e2e","signature_id":"73ffdde7c35ef7b4f90a0d9d0266878e677b3024d0af73c677985cd106c56138","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00280b770\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00280b770\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101554844631306240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7019,"pr_state":"closed","pr_sha":"42b2541179ea46321dd5a605f46f5fc661e80081","final_merged_sha":"42b2541179ea46321dd5a605f46f5fc661e80081","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T06:11:34Z","started_at":"2026-09-20T06:11:34Z","completed_at":"2026-09-20T07:52:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T07:58:53.058614227Z","timing_metadata_checked_at":"2026-09-20T07:58:53.058614227Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T07:58:53.062053816Z","region_metadata_checked_at":"2026-09-20T07:58:53.062053816Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101543782393581568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"closed","pr_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","final_merged_sha":"393527983bbf3810fedd8ca66cc790d3d1526978","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T05:27:37Z","started_at":"2026-09-20T05:27:37Z","completed_at":"2026-09-20T07:27:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T07:37:53.435532307Z","timing_metadata_checked_at":"2026-09-20T07:37:53.435532307Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T07:37:53.438474796Z","region_metadata_checked_at":"2026-09-20T07:37:53.438474796Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101522358387871744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7019,"pr_state":"closed","pr_sha":"42b2541179ea46321dd5a605f46f5fc661e80081","final_merged_sha":"42b2541179ea46321dd5a605f46f5fc661e80081","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-20T04:02:00Z","started_at":"2026-09-20T04:02:00Z","completed_at":"2026-09-20T06:05:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T06:22:51.333529358Z","timing_metadata_checked_at":"2026-09-20T06:22:51.333529358Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T06:22:51.336988347Z","region_metadata_checked_at":"2026-09-20T06:22:51.336988347Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"469b589d40cc288317ac2017d05939d5957a51aa85041b4c230ae7113cc8c78c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101522358387871744","occurred_at":"2026-09-20T04:02:00Z","failed_at":"e2e","signature_id":"07b9cc47f1337561bab912a04c50994e47b493cb94f69944313b85e20e5e8827","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001f2540\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-g8jlwfczq87l\" to finish updating: GET https://rp.j7871744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001f2540\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-g8jlwfczq87l\" to finish updating: GET https://rp.j7871744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7024/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101522356693372928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7024,"pr_state":"closed","pr_sha":"7a275cf861815aa8598c8a212a1d953a6d2da491","final_merged_sha":"7a275cf861815aa8598c8a212a1d953a6d2da491","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-20T04:01:55Z","started_at":"2026-09-20T04:01:55Z","completed_at":"2026-09-20T06:17:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-20T06:33:46.603267219Z","timing_metadata_checked_at":"2026-09-20T06:33:46.603267219Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-20T06:33:46.606884807Z","region_metadata_checked_at":"2026-09-20T06:33:46.606884807Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}