{"meta":{"date":"2026-10-13","timezone":"UTC","generated_at":"2026-11-01T07:43:31Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":81,"failed_runs":57,"runs_with_occurrences":57,"runs_with_failure_pattern_matches":50,"runs_with_unmatched_occurrences":7,"runs_without_occurrence_rows":24,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110153446450532352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7495,"pr_state":"open","pr_sha":"3c936e16bdf98e15f0724566b1efeaef8d31a693","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T23:39:20Z","started_at":"2026-10-13T23:39:20Z","completed_at":"2026-10-14T02:04:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-14T02:20:52.775545315Z","timing_metadata_checked_at":"2026-10-14T02:20:52.775545315Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-14T02:20:52.778674205Z","region_metadata_checked_at":"2026-10-14T02:20:52.778674205Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"85df438ba4d34cc3162707d37ac757de08b84fc5b3a996796a96e2b8f8afc0fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110153446450532352","occurred_at":"2026-10-13T23:39:20Z","failed_at":"alert","signature_id":"3845ce22d709f7d6f88c6b78d29a932df4b7e3435bd9d98ded715b56bb04b04e","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j0532352 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-14T00:40:15Z\n Ended: 2026-10-14T01:10:12Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired"}},{"row_id":"a348021343c7c02d5296301851bf36052ff0b978c48a8dd70b0aebd022907822","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110153446450532352","occurred_at":"2026-10-13T23:39:20Z","failed_at":"alert","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j0532352 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-14T00:46:13Z\n Ended: 2026-10-14T01:18:14Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110148238915932160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7469,"pr_state":"closed","pr_sha":"a2d7fd940b1ba037c80cee86cf5b380e84ab4193","final_merged_sha":"a2d7fd940b1ba037c80cee86cf5b380e84ab4193","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-13T23:18:38Z","started_at":"2026-10-13T23:18:38Z","completed_at":"2026-10-14T01:36:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-14T01:48:57.737867585Z","timing_metadata_checked_at":"2026-10-14T01:48:57.737867585Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-14T01:48:57.741625874Z","region_metadata_checked_at":"2026-10-14T01:48:57.741625874Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110141907093950464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"open","pr_sha":"aec623ca6805f7617d60f2a1fa0a7483e73e8b8e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T22:53:29Z","started_at":"2026-10-13T22:53:29Z","completed_at":"2026-10-14T00:57:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-14T01:07:28.666334644Z","timing_metadata_checked_at":"2026-10-14T01:07:28.666334644Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-14T01:07:28.669641833Z","region_metadata_checked_at":"2026-10-14T01:07:28.669641833Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3cd0aa9260415d73e0580486b5b4d580691177cf5e984eb36efff5e9258bbbc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110141907093950464","occurred_at":"2026-10-13T22:53:29Z","failed_at":"e2e","signature_id":"b866590f9b46bfea6312f2e0fb04cbc1e34def7eb9a707daec346d608f89ac23","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x283328822c40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53294-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53294-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:364]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x283328822c40\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53294-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53294-\u003e172.30.0.10:53: read: connection refused\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110137118025060352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6712,"pr_state":"open","pr_sha":"9bfd9800d00b7d8a9afc6860ec1c4a34d8307a77","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T22:34:27Z","started_at":"2026-10-13T22:34:27Z","completed_at":"2026-10-14T00:47:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-14T00:57:27.945435462Z","timing_metadata_checked_at":"2026-10-14T00:57:27.945435462Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-14T00:57:27.948922551Z","region_metadata_checked_at":"2026-10-14T00:57:27.948922551Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4a39971ffbec41970cb0cf5ffcd431943f85504bbe86acef3d9a069ebd08ac68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110137118025060352","occurred_at":"2026-10-13T22:34:27Z","failed_at":"e2e","signature_id":"94d1e13589455fbe3faae0d56a4527d0080bf018f5cc0df414cf6c40b010f2da","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-wvbxxwdg6hgk/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-wvbxxwdg6hgk/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-c5gl2m96t572/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7509,"pr_state":"closed","pr_sha":"bb4f9003b3aaf3d6ab0d1b5d4c00c9f68e0d6a9d","final_merged_sha":"bb4f9003b3aaf3d6ab0d1b5d4c00c9f68e0d6a9d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T22:12:28Z","started_at":"2026-10-13T22:12:28Z","completed_at":"2026-10-14T00:28:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-14T00:46:57.556273549Z","timing_metadata_checked_at":"2026-10-14T00:46:57.556273549Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-14T00:46:57.561406033Z","region_metadata_checked_at":"2026-10-14T00:46:57.561406033Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"22d889424b57391f0f7cb87415e62ce6722cbe0ebceb4fbf99b410203acccc48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","failed_at":"alert","signature_id":"a9ea08e80c10006881b8dcdd8e6fcea6f11b3f2c9b0ce3779517a03365f66b46","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T23:17:32Z\n Ended: 2026-10-13T23:44:30Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-l4xqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T23:17:32Z\n Ended: 2026-10-13T23:43:31Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-l4xqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"b8b58dcdd4d4127a4984478b3f02b127e8e1fead4334c9b2acc44f25d482cab7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","failed_at":"alert","signature_id":"81e6aedf0da4f27efde8cb67872b6903101fe3182111cf67d2dfdc5646317ecb","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T23:32:29Z\n Ended: 2026-10-13T23:45:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-l4xqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T23:32:29Z\n Ended: 2026-10-13T23:45:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-l4xqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"cb154833cfd1d219aef64577a7d70726b93e54a984f61877f0c18224f8bbfc13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","failed_at":"e2e","signature_id":"3b8fba5a1aa06016e6c6895bc3863f7f6cf359856948127e7c81a165583cfcd6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8fsm9cfscsm2/cluster-candidate-5-0-gcnmsr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x31dfd85b2460\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gcnmsr: failed waiting for cluster=\"cluster-candidate-5-0-gcnmsr\" in resourcegroup=\"rg-candidate-5-0-8fsm9cfscsm2\" to finish creating: GET https://rp.j3394432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\",\n \"name\": \"b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:56:29.235009494Z\",\n \"endTime\": \"2026-10-13T23:25:36.777262748Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3394432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\n correlation ID: fd6f57aa-c468-479d-a56c-8186a1e8023b\n client request ID: 427c0be1-3302-4f9f-a711-14e8b93546be\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-8fsm9cfscsm2/cluster-candidate-5-0-gcnmsr should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x31dfd85b2460\u003e: \n failed to create HCP cluster cluster-candidate-5-0-gcnmsr: failed waiting for cluster=\"cluster-candidate-5-0-gcnmsr\" in resourcegroup=\"rg-candidate-5-0-8fsm9cfscsm2\" to finish creating: GET https://rp.j3394432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\",\n \"name\": \"b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:56:29.235009494Z\",\n \"endTime\": \"2026-10-13T23:25:36.777262748Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3394432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b3f80f07-6711-40dc-aa63-cd1bb0fc5de6\n correlation ID: fd6f57aa-c468-479d-a56c-8186a1e8023b\n client request ID: 427c0be1-3302-4f9f-a711-14e8b93546be\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f90705708a86433bd8031b71669ae79de931716b38a6e189ec975484d20bd509","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7509/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131584773394432","occurred_at":"2026-10-13T22:12:28Z","failed_at":"alert","signature_id":"ab3005e87ad04ff712d959705038bb2a3a5aadeb7a3d7406d2362bf8f67d8cf4","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T23:20:28Z\n Ended: 2026-10-13T23:48:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"77588538-1895-449d-bb34-5ee7b9aabbd3\"\n Description: Pod ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-13T23:20:28Z\n Ended: 2026-10-13T23:48:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3394432-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3394432/providers/microsoft.monitor/accounts/hcps-j3394432\", namespace=\"ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"77588538-1895-449d-bb34-5ee7b9aabbd3\"\n Description: Pod ocm-arohcpci01-2te6i7gkbepqvjtjr55req1psul8267o-l7d9p9t3f9q6b2m/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110131387603357696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7401,"pr_state":"closed","pr_sha":"0cf013ee7a82ebd50ca7f8532f6a14f30a890da6","final_merged_sha":"c5a41fd2de605bcf603eae1d1ad16d571aaa93a9","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T22:11:41Z","started_at":"2026-10-13T22:11:41Z","completed_at":"2026-10-14T00:21:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-14T00:36:49.386470835Z","timing_metadata_checked_at":"2026-10-14T00:36:49.386470835Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-14T00:36:49.389944525Z","region_metadata_checked_at":"2026-10-14T00:36:49.389944525Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110129974156464128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7477,"pr_state":"closed","pr_sha":"ebcb034612089df95d850e0aa0c646d1016baa22","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T22:06:04Z","started_at":"2026-10-13T22:06:04Z","completed_at":"2026-10-14T00:20:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-14T00:36:49.409130667Z","timing_metadata_checked_at":"2026-10-14T00:36:49.409130667Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-14T00:36:49.417408743Z","region_metadata_checked_at":"2026-10-14T00:36:49.417408743Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1369c76103725cd390675a14a699543174b8d967d9d5b61f317b38ff96799bdd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110129974156464128","occurred_at":"2026-10-13T22:06:04Z","failed_at":"e2e","signature_id":"91cd8349ac9ddd72fa8ca1ae3cea72c2db06bc446d6e997ab59539aff1b80c1e","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*errors.errorString | 0x374a5e1794b0\u003e: \n IngressController loadBalancer config is nil\n {\n s: \"IngressController loadBalancer config is nil\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*errors.errorString | 0x374a5e1794b0\u003e: \n IngressController loadBalancer config is nil\n {\n s: \"IngressController loadBalancer config is nil\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed30ba2b16f76586a1f98b0129dc60d2a7c651f09c53499a5a21b5f5dff0e754","failure_pattern":"IngressController loadBalancer config is nil","search_query":"IngressController loadBalancer config is nil"}},{"row_id":"8755f5344ba9b8e0a01a07d8ea5582b5160621abb9fe09cd4efada5d033cd00a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110129974156464128","occurred_at":"2026-10-13T22:06:04Z","failed_at":"e2e","signature_id":"1dff6fd88e7b67ee0e3d3f634407f5c779eb894a4458935f87ace999c5c319de","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*errors.errorString | 0x284568f46bc0\u003e: \n IngressController loadBalancer config is nil\n {\n s: \"IngressController loadBalancer config is nil\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:198]: default ingress certificate scenario requires public ingress\nExpected success, but got an error:\n \u003c*errors.errorString | 0x284568f46bc0\u003e: \n IngressController loadBalancer config is nil\n {\n s: \"IngressController loadBalancer config is nil\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ed30ba2b16f76586a1f98b0129dc60d2a7c651f09c53499a5a21b5f5dff0e754","failure_pattern":"IngressController loadBalancer config is nil","search_query":"IngressController loadBalancer config is nil"}},{"row_id":"c9754b828c2b538d5cdad2baa476d571bcfad55b93c4e9b1112c1b2bafb32aa3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110129974156464128","occurred_at":"2026-10-13T22:06:04Z","failed_at":"e2e","signature_id":"694e2d897de6010d0383c8b74dec7d08484cd381d7c5e3824ebcf322dba221fb","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b033ad20c80\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-13T22:55:07.0962439Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T22:55:07.0962439Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2b033ad20c60\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T22:55:07.0962439Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x2b0339c5a8c0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x2b033ad20c40\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T22:55:07.0962439Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x2b033bf13d10\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x2b033ad20c80\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-13T22:55:07.0962439Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T22:55:07.0962439Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x2b033ad20c60\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T22:55:07.0962439Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x2b0339c5a8c0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x2b033ad20c40\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5d7srqg64hdt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T22:55:07.0962439Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x2b033bf13d10\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["ed30ba2b16f76586a1f98b0129dc60d2a7c651f09c53499a5a21b5f5dff0e754","fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110123903912251392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7488,"pr_state":"closed","pr_sha":"e343803f365d228cc3e0070ed414c588a26b5e9f","final_merged_sha":"e343803f365d228cc3e0070ed414c588a26b5e9f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T21:41:57Z","started_at":"2026-10-13T21:41:57Z","completed_at":"2026-10-13T23:59:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-14T00:16:24.823890241Z","timing_metadata_checked_at":"2026-10-14T00:16:24.823890241Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-14T00:16:24.827176731Z","region_metadata_checked_at":"2026-10-14T00:16:24.827176731Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0977a15f45efd35cfa8ad86beeceb1009098bdd2cbbada5f61969526ee48c2e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110123903912251392","occurred_at":"2026-10-13T21:41:57Z","failed_at":"e2e","signature_id":"8b0aac51428f6336203e5c99b582b7417a3d1fd3f1323459c75ebe8411a4f273","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 604.992s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39f5f91e0c80\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\n let _endTime = datetime(2026-10-13T23:09:06.1094687Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T23:09:29.6842855Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\", ActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", SubActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x39f5f8768288\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x39f5f91e0c60\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0c40\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0ba0\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0b60\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x39f5f6b7ee00\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 604.992s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39f5f91e0c80\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\n let _endTime = datetime(2026-10-13T23:09:06.1094687Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T23:09:29.6842855Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\", ActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", SubActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\", ActivityStack:\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x39f5f8768288\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x39f5f91e0c60\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0c40\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0ba0\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39f5f91e0b60\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06.1094684Z);\\nlet _endTime = datetime(2026-10-13T23:09:06.1094687Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-nvqd2j9ncsfl'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T23:09:29.6842855Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea\\\", ActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", SubActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"0d59e32d-82d3-46a4-953e-e100fb2ccef8\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;6eceef39-9526-457e-97d9-feedc18102ea ARID=0d59e32d-82d3-46a4-953e-e100fb2ccef8 \u003e GW.Http.CallContext/0d59e32d-82d3-46a4-953e-e100fb2ccef8)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x39f5f6b7ee00\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8baf31258527a9c37481b02b6c4e240b9ba893342f978c453787fc7905c9d95b","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T23:09:06"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8baf31258527a9c37481b02b6c4e240b9ba893342f978c453787fc7905c9d95b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110123884639424512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7248,"pr_state":"closed","pr_sha":"29ee7d9651963ed9c17688c509801cba7771db4c","final_merged_sha":"29ee7d9651963ed9c17688c509801cba7771db4c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T21:41:52Z","started_at":"2026-10-13T21:41:52Z","completed_at":"2026-10-13T23:56:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-14T00:16:24.723330856Z","timing_metadata_checked_at":"2026-10-14T00:16:24.723330856Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-14T00:16:24.727288443Z","region_metadata_checked_at":"2026-10-14T00:16:24.727288443Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"22ef7ec581bca61e456c59c2552b28ecbc10c7643c6d6d829ea8d46e8fb5b241","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110123884639424512","occurred_at":"2026-10-13T21:41:52Z","failed_at":"e2e","signature_id":"7abba32d1386c404a65777a77a6155bf3cb76d06da7c18bd2cc1819fac47609a","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2b2721dce700\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xq96765qgwxz\" to finish creating: GET https://rp.j9424512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\",\n \"name\": \"70cccd80-322a-41c6-aef6-dff76071386f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:47:02.133519323Z\",\n \"endTime\": \"2026-10-13T23:16:06.91066671Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-xq96765qgwxz--managed/providers/Microsoft.Compute/virtualMachines/x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-xq96765qgwxz--managed/providers/Microsoft.Compute/virtualMachines/x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9424512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\n correlation ID: 10fb6c26-b492-4ad7-bd1d-956c35d26f43\n client request ID: 62230a28-0708-4526-b5b0-b1e90cbbcfa0\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2b2721dce700\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xq96765qgwxz\" to finish creating: GET https://rp.j9424512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\",\n \"name\": \"70cccd80-322a-41c6-aef6-dff76071386f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:47:02.133519323Z\",\n \"endTime\": \"2026-10-13T23:16:06.91066671Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-xq96765qgwxz--managed/providers/Microsoft.Compute/virtualMachines/x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-xq96765qgwxz--managed/providers/Microsoft.Compute/virtualMachines/x8s4a0b7o3h5z7m-np-delete-main-8qs7b-hs6qs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9424512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/70cccd80-322a-41c6-aef6-dff76071386f\n correlation ID: 10fb6c26-b492-4ad7-bd1d-956c35d26f43\n client request ID: 62230a28-0708-4526-b5b0-b1e90cbbcfa0","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dpn7kns9h5s6\" to finish creating: GET https://rp"}},{"row_id":"e29b94d68bc46da729a2773d8f5e6a6d8649f482851b13b78542fb2f65fede07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110123884639424512","occurred_at":"2026-10-13T21:41:52Z","failed_at":"e2e","signature_id":"983126b83f0ae7565666b720c4dde22c9930fd7261ca4bcc8961d600d3ad5a6a","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-qrdm5stt6s2r/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-qrdm5stt6s2r/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-c5gl2m96t572/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110117744396472320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7278,"pr_state":"open","pr_sha":"93f484cb2f1316bfaa43f57d0ce4a3a6838f96d3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T21:17:25Z","started_at":"2026-10-13T21:17:25Z","completed_at":"2026-10-13T23:32:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T23:45:07.034644077Z","timing_metadata_checked_at":"2026-10-13T23:45:07.034644077Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T23:45:07.039901861Z","region_metadata_checked_at":"2026-10-13T23:45:07.039901861Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7495,"pr_state":"open","pr_sha":"3c936e16bdf98e15f0724566b1efeaef8d31a693","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T21:09:22Z","started_at":"2026-10-13T21:09:22Z","completed_at":"2026-10-13T23:18:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T23:34:43.646277803Z","timing_metadata_checked_at":"2026-10-13T23:34:43.646277803Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T23:34:43.649078594Z","region_metadata_checked_at":"2026-10-13T23:34:43.649078594Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"261ee3bfba66c99d861096da4e2415bfd033c1e0290391694411ccdaaa2d2236","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","failed_at":"alert","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j2816128 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T22:09:34Z\n Ended: 2026-10-13T22:42:14Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired"}},{"row_id":"46561879cd293084056145d894aa63164b9c403dcd17f135cb534d94c7950897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","failed_at":"alert","signature_id":"5df3df51c4f1a79ff78be7277fb6033fa17feb7d65772f7a0900bc893660bf39","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T22:19:01Z\n Ended: 2026-10-13T22:28:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2816128-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2te5m776ut8ij1hl0ecm8bji2rolvh54-y4r5q8u1c0g8x2m\", pod=\"router-5654cb9d4d-gd8l4\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2te5m776ut8ij1hl0ecm8bji2rolvh54-y4r5q8u1c0g8x2m/router-5654cb9d4d-gd8l4 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"5002924b0356dfd7c9fa161ba3b01065801aaa44f172c4b1ef2e684d7fe29611","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","failed_at":"alert","signature_id":"3845ce22d709f7d6f88c6b78d29a932df4b7e3435bd9d98ded715b56bb04b04e","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j2816128 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T22:03:21Z\n Ended: 2026-10-13T22:29:18Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired"}},{"row_id":"7e5b49251023ecccdb59e7fd905937e733fee898248ee083457560a671a3cec1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110115702512816128","occurred_at":"2026-10-13T21:09:22Z","failed_at":"e2e","signature_id":"3a7b7f1faaa1631909c7c15abf2900e7c8f3a919926de8c9c9b43b605988f0ed","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1a46c002a160\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bvw879vk9kf2\" to finish creating: GET https://rp.j2816128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\",\n \"name\": \"96d5a19d-7a94-4c90-b6f3-78fa73f80a64\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:07:49.726995766Z\",\n \"endTime\": \"2026-10-13T22:36:58.709896645Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bvw879vk9kf2--managed/providers/Microsoft.Compute/virtualMachines/w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bvw879vk9kf2--managed/providers/Microsoft.Compute/virtualMachines/w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2816128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\n correlation ID: 79ec83c3-f7b9-41f5-baf4-9c117904430d\n client request ID: fa7965b1-cdd5-4c6b-bbfc-bfe81d721d6f\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1a46c002a160\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bvw879vk9kf2\" to finish creating: GET https://rp.j2816128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\",\n \"name\": \"96d5a19d-7a94-4c90-b6f3-78fa73f80a64\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T22:07:49.726995766Z\",\n \"endTime\": \"2026-10-13T22:36:58.709896645Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bvw879vk9kf2--managed/providers/Microsoft.Compute/virtualMachines/w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bvw879vk9kf2--managed/providers/Microsoft.Compute/virtualMachines/w4b8j4s5n4t4f5w-np-delete-main-k5rps-jn5mr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2816128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96d5a19d-7a94-4c90-b6f3-78fa73f80a64\n correlation ID: 79ec83c3-f7b9-41f5-baf4-9c117904430d\n client request ID: fa7965b1-cdd5-4c6b-bbfc-bfe81d721d6f","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dpn7kns9h5s6\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110112563529256960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7469,"pr_state":"closed","pr_sha":"a2d7fd940b1ba037c80cee86cf5b380e84ab4193","final_merged_sha":"a2d7fd940b1ba037c80cee86cf5b380e84ab4193","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T20:56:53Z","started_at":"2026-10-13T20:56:53Z","completed_at":"2026-10-13T23:13:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T23:24:29.600472433Z","timing_metadata_checked_at":"2026-10-13T23:24:29.600472433Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T23:24:29.603717523Z","region_metadata_checked_at":"2026-10-13T23:24:29.603717523Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"410f41c5fbaa0eb91dc987b58cb31af3cae49eabc2858a125dfff1e3cfd13717","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110112563529256960","occurred_at":"2026-10-13T20:56:53Z","failed_at":"e2e","signature_id":"efb9c37e965d24b97d7e8ab613bd6ca3432ad4802836e8dcd3a2df165e5003e5","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x389cde3b8e20\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-13T21:46:20.5229327Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T21:46:20.5229327Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x389cde3b8de0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T21:46:20.5229327Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x389cdd5785a0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x389cde3b8d60\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T21:46:20.5229327Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x389cdf50a8a0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x389cde3b8e20\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-13T21:46:20.5229327Z);\n let desires = cosmosResourceSnapshots\n | where resourceID startswith strcat(target, \"/\")\n | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n | where isempty(content.deletionTimestamp)\n | extend manifest = content.properties.status.kubeContent\n | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n let fleet = cosmosResourceSnapshots\n | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n | where isempty(content.deletionTimestamp)\n | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n | where isnotempty(clusterName)\n | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n let hcp = desires\n | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n | distinct environment, region, cluster, namespace;\n let inventory = materialize(kubernetesResourceSnapshots\n | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n let pods = inventory\n | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n let deployments = inventory\n | where objectKind == \"Deployment\" and name == \"router\"\n | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n let summaries = containerLogs\n | where timestamp \u003e= start and container_name == \"swift-recorder\"\n | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n | join kind=leftsemi hcp on environment, region, cluster, namespace\n | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n hcp\n | join kind=leftouter deployments on environment, region, cluster, namespace\n | join kind=leftouter counts on environment, region, cluster, namespace\n | join kind=leftouter pods on environment, region, cluster, namespace\n | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n | order by Scope asc, UID asc\n | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.007s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T21:46:20.5229327Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x389cde3b8de0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T21:46:20.5229327Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x389cdd5785a0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x389cde3b8d60\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qpjkt6pf45sb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-13T21:46:20.5229327Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| join kind=leftsemi hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x389cdf50a8a0\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded"}},{"row_id":"f090cf9f191dd68a5c0e7b53ac7460b01191d43f9c741fbc9d30aa02402d87ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110112563529256960","occurred_at":"2026-10-13T20:56:53Z","failed_at":"e2e","signature_id":"814ddd39348331a533c35437b54ed67f25a7378b56607756b2224dbb1c825092","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:139]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2515216a5b20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dpn7kns9h5s6\" to finish creating: GET https://rp.j9256960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02ba10f3-0e8a-4716-94ab-581f07c35309\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02ba10f3-0e8a-4716-94ab-581f07c35309\",\n \"name\": \"02ba10f3-0e8a-4716-94ab-581f07c35309\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T21:54:35.192495222Z\",\n \"endTime\": \"2026-10-13T22:23:37.481089236Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-fc8dz-zl2sj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-dpn7kns9h5s6--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-fc8dz-zl2sj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-fc8dz-zl2sj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-dpn7kns9h5s6--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-fc8dz-zl2sj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9256960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02ba10f3-0e8a-4716-94ab-581f07c35309\n correlation ID: 69c597ca-4a5d-45f4-be8b-542aa16b2e4b\n client request ID: 15111503-9313-4b39-bc7c-c0bfc7ad586b\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:139]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2515216a5b20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dpn7kns9h5s6\" to finish creating: GET https://rp.j9256960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02ba10f3-0e8a-4716-94ab-581f07c35309\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02ba10f3-0e8a-4716-94ab-581f07c35309\",\n \"name\": \"02ba10f3-0e8a-4716-94ab-581f07c35309\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T21:54:35.192495222Z\",\n \"endTime\": \"2026-10-13T22:23:37.481089236Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-fc8dz-zl2sj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-dpn7kns9h5s6--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-fc8dz-zl2sj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-fc8dz-zl2sj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-dpn7kns9h5s6--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-fc8dz-zl2sj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9256960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02ba10f3-0e8a-4716-94ab-581f07c35309\n correlation ID: 69c597ca-4a5d-45f4-be8b-542aa16b2e4b\n client request ID: 15111503-9313-4b39-bc7c-c0bfc7ad586b","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dpn7kns9h5s6\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110111376990015488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7483,"pr_state":"open","pr_sha":"56f1c99a1d7838e70007cd9f500de1aca7c582d9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T20:52:10Z","started_at":"2026-10-13T20:52:10Z","completed_at":"2026-10-13T23:17:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T23:34:42.47079242Z","timing_metadata_checked_at":"2026-10-13T23:34:42.47079242Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T23:34:42.475329905Z","region_metadata_checked_at":"2026-10-13T23:34:42.475329905Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110111278457425920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6807,"pr_state":"open","pr_sha":"ad91b7f706156ff9b8f595b00a6a8d6f2cc97a8d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T20:51:47Z","started_at":"2026-10-13T20:51:47Z","completed_at":"2026-10-13T23:12:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T23:24:28.273474382Z","timing_metadata_checked_at":"2026-10-13T23:24:28.273474382Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T23:24:28.276983671Z","region_metadata_checked_at":"2026-10-13T23:24:28.276983671Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7253/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110107953607479296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7253,"pr_state":"open","pr_sha":"72572322a275d718fe227782b5f5867f2a389729","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T20:38:27Z","started_at":"2026-10-13T20:38:27Z","completed_at":"2026-10-13T22:59:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T23:03:45.699261455Z","timing_metadata_checked_at":"2026-10-13T23:03:45.699261455Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T23:03:45.702478145Z","region_metadata_checked_at":"2026-10-13T23:03:45.702478145Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110099886413189120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7480,"pr_state":"closed","pr_sha":"2965e4d41141746364c56236dd2455f15329afaa","final_merged_sha":"230a5d27c416299f628f0667ba08deeea65c4c88","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T20:04:06Z","started_at":"2026-10-13T20:04:06Z","completed_at":"2026-10-13T22:29:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T22:39:55.618949517Z","timing_metadata_checked_at":"2026-10-13T22:39:55.618949517Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T22:39:55.622567306Z","region_metadata_checked_at":"2026-10-13T22:39:55.622567306Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"ac9ce76c02dd8dec522debca73db1fddcae5c94c1d41794db010ce593bbffeac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110099886413189120","occurred_at":"2026-10-13T20:04:06Z","failed_at":"e2e","signature_id":"e835da002c88402841b37e84e6cf6143f8cac4fd26e56433266edb67a38bff39","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n \u003c*exported.ResponseError | 0x3788bb56a390\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n correlation ID: 135296a0-8b8d-4ae4-9dc1-dcc858885005\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:108]: failed to get service managed identity\nUnexpected error:\n \u003c*exported.ResponseError | 0x3788bb56a390\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-02-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\n correlation ID: 135296a0-8b8d-4ae4-9dc1-dcc858885005\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429"}},{"row_id":"ff114b820011b60adb878e0e7661c7da824d6a6da6f195d74c8bd7c2b9a5664b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110099886413189120","occurred_at":"2026-10-13T20:04:06Z","failed_at":"alert","signature_id":"ab4dcac27ea15b642051449befb4d2d369e9937091c6b496b59f54afa24883cd","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T21:38:41Z\n Ended: 2026-10-13T21:44:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3189120-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2te4tunmapm3qrg9otufrskbuf4u75vg-h7o1o8e4n0s6g9g\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2te4tunmapm3qrg9otufrskbuf4u75vg-h7o1o8e4n0s6g9g/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110098887174787072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7503,"pr_state":"closed","pr_sha":"dd38437739ccfb5fe4d5d4bd849164c65a9bad17","final_merged_sha":"dd38437739ccfb5fe4d5d4bd849164c65a9bad17","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T19:57:04Z","started_at":"2026-10-13T19:57:04Z","completed_at":"2026-10-13T22:42:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T22:53:03.488694511Z","timing_metadata_checked_at":"2026-10-13T22:53:03.488694511Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T22:53:03.491579602Z","region_metadata_checked_at":"2026-10-13T22:53:03.491579602Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"24ce56d4571cea5e5e0e64ff771cd520cc2dce3acf27a1d204383049bc3c64cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110098887174787072","occurred_at":"2026-10-13T19:57:04Z","failed_at":"e2e","signature_id":"181ae70b8645dc86d286ecef9ed77dae1b64d0e1fdc182c681d11979a0362e0d","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x332d8f70c120\u003e: \n resource group \"rg-hs-pre--pvfm6ns8qtjf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-6j9rv6\" in resourcegroup=\"rg-hs-pre--pvfm6ns8qtjf\" to finish deleting: GET https://rp.j4787072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\",\n \"name\": \"8d2fc814-64dd-4880-8292-2fd063dab951\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T21:38:32.56326477Z\",\n \"endTime\": \"2026-10-13T22:02:34.567780855Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4787072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\n correlation ID: c238bf11-ea0d-4948-b2da-1eaa7be46473\n client request ID: bbe62e16-afcb-49d8-bc81-105532d0b70b\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x332d8f70c120\u003e: \n resource group \"rg-hs-pre--pvfm6ns8qtjf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-6j9rv6\" in resourcegroup=\"rg-hs-pre--pvfm6ns8qtjf\" to finish deleting: GET https://rp.j4787072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\",\n \"name\": \"8d2fc814-64dd-4880-8292-2fd063dab951\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T21:38:32.56326477Z\",\n \"endTime\": \"2026-10-13T22:02:34.567780855Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4787072.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d2fc814-64dd-4880-8292-2fd063dab951\n correlation ID: c238bf11-ea0d-4948-b2da-1eaa7be46473\n client request ID: bbe62e16-afcb-49d8-bc81-105532d0b70b\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9504e9808966726f88f1e3aa95a816f856740893c0a6e0e93af5e45f48daaf78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110098887174787072","occurred_at":"2026-10-13T19:57:04Z","failed_at":"e2e","signature_id":"c26a9d223de96e2bc27095ea2774785d4152120405397e9ea2ecd9e4d14ed85b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-5wrsns\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x9f731ad0ea0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-mve2kajh3vwg6' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n correlation ID: 547339b0-2310-46d9-9b61-b7c398526e80\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-23-5wrsns\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x9f731ad0ea0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"VaultAlreadyExists\\\",\\r\\n \\\"message\\\": \\\"The vault name 'cust-kv-mve2kajh3vwg6' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-23-gn9flc7pzw5b/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-23-5wrsns-td2dnf/operationStatuses/08584096807961245220\n correlation ID: 547339b0-2310-46d9-9b61-b7c398526e80\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"9abdc928e59b239758217615dfa8639050c65e244113a8ca92a9296e27c02b88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110098887174787072","occurred_at":"2026-10-13T19:57:04Z","failed_at":"alert","signature_id":"4587179fd233fc65eb9fdb22388faacf6ec23475c73d8a87452ef9bb1dc63ec6","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T22:15:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4787072-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.80:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4787072/providers/microsoft.monitor/accounts/hcps-j4787072\", namespace=\"ocm-arohcpci01-2te530rhjqvehkhg5fuooml9ocotbb8a-j7w4j0s2f4d4g1g\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vkmx4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te530rhjqvehkhg5fuooml9ocotbb8a-j7w4j0s2f4d4g1g/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-13T22:15:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4787072-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.80:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4787072/providers/microsoft.monitor/accounts/hcps-j4787072\", namespace=\"ocm-arohcpci01-2te530rhjqvehkhg5fuooml9ocotbb8a-j7w4j0s2f4d4g1g\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vkmx4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te530rhjqvehkhg5fuooml9ocotbb8a-j7w4j0s2f4d4g1g/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0d7a8e5dc2d8e7103fa834285288a1f3b3954abe71b752dee4412a0b1b6737d5","13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110095077329080320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7313,"pr_state":"closed","pr_sha":"02b7511e30cc16674b6938c06b7b2aec2ccdcecf","final_merged_sha":"02b7511e30cc16674b6938c06b7b2aec2ccdcecf","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-13T19:41:56Z","started_at":"2026-10-13T19:41:56Z","completed_at":"2026-10-13T22:19:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T22:29:36.123427584Z","timing_metadata_checked_at":"2026-10-13T22:29:36.123427584Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T22:29:36.130020462Z","region_metadata_checked_at":"2026-10-13T22:29:36.130020462Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110096712633683968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7504,"pr_state":"open","pr_sha":"c0e2f18ace51d06f9bcf47af05e7fa0bec608ae0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T19:39:31Z","started_at":"2026-10-13T19:39:31Z","completed_at":"2026-10-13T20:54:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T21:06:31.664688717Z","timing_metadata_checked_at":"2026-10-13T21:06:31.664688717Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-13T21:06:31.667687706Z","region_metadata_checked_at":"2026-10-13T22:08:34.586189803Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a20a88febe9df0fa21f2e2cbd318b64c9eab6062a38ca6fe4d15e9e3747c3d9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110096712633683968","occurred_at":"2026-10-13T19:39:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110091526691360768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7492,"pr_state":"closed","pr_sha":"5bf7c6eb96af486e0a3ec4a5df97a32a7e503808","final_merged_sha":"5bf7c6eb96af486e0a3ec4a5df97a32a7e503808","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T19:28:19Z","started_at":"2026-10-13T19:28:19Z","completed_at":"2026-10-13T22:04:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T22:18:42.604216659Z","timing_metadata_checked_at":"2026-10-13T22:18:42.604216659Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T22:18:42.608012847Z","region_metadata_checked_at":"2026-10-13T22:18:42.608012847Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"11962a84c45aebf634cd9a2ad07d14a72c44569aef2f085a229faba1e8f15e6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110091526691360768","occurred_at":"2026-10-13T19:28:19Z","failed_at":"e2e","signature_id":"f25671b9f4bd359b018cb9e9ac954a66d4ed6290a26895e1fd925ac9f22976fe","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.008s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39ffd3acaac0\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\n let _endTime = datetime(2026-10-13T21:06:23.2601111Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T21:06:33.6034532Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\", ActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", SubActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityStack:\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x39ffd2546540\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x39ffd3acaaa0\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd3acaa80\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd0047e00\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd0047dc0\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x39ffd0960100\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.008s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x39ffd3acaac0\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\n let _endTime = datetime(2026-10-13T21:06:23.2601111Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-13T21:06:33.6034532Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\", ActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", SubActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"91f407a2-0a4c-43e0-acd3-c6532e675296\", ActivityStack:\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x39ffd2546540\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x39ffd3acaaa0\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd3acaa80\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd0047e00\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x39ffd0047dc0\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23.2601110Z);\\nlet _endTime = datetime(2026-10-13T21:06:23.2601111Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-tq6r7tbm7n4m'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-13T21:06:33.6034532Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558\\\", ActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", SubActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"91f407a2-0a4c-43e0-acd3-c6532e675296\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;467201e3-34eb-4ddd-9d6d-32112ea40558 ARID=91f407a2-0a4c-43e0-acd3-c6532e675296 \u003e GW.Http.CallContext/91f407a2-0a4c-43e0-acd3-c6532e675296)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x39ffd0960100\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"deea4d7ee03e8432bcb988590b2236f74b08241de4cdde1789d1247ae2f6a03f","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-12T21:06:23"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["deea4d7ee03e8432bcb988590b2236f74b08241de4cdde1789d1247ae2f6a03f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110091971564408832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7502,"pr_state":"closed","pr_sha":"46b8ad997c74f398f17e0d97e729473b9c3cfbc6","final_merged_sha":"4e388171c7cac99a0d73a3ae18b94e46f3e3866f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T19:26:38Z","started_at":"2026-10-13T19:26:38Z","completed_at":"2026-10-13T20:17:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T20:34:19.490691482Z","timing_metadata_checked_at":"2026-10-13T20:34:19.490691482Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T20:34:19.493264774Z","region_metadata_checked_at":"2026-10-13T20:34:19.493264774Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0dc4ea4e9583011928c15fea520d83fa9ec782d3f409f1c5a0462f676340f216","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110091971564408832","occurred_at":"2026-10-13T19:26:38Z","failed_at":"provision","signature_id":"9ca8c9ef2c96a2d57799808a4c23411872eadb74fb13e040cc050d45a0ce4d60","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-13T20:10:59.938Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 description=\"Step swift-vnet\\n Kind: Shell\\n Command: ./swift-vnet.sh\\n\"\ntime=2026-10-13T20:11:00.569Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: Failed to load or parse file /tmp/azure-cli-config-533168303/azureProfile.json. It will be overridden by default settings.\\nERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc","failure_pattern":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","search_query":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110083918836273152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7505,"pr_state":"closed","pr_sha":"931ee687540c9a77bd76ea5c902076e97440d479","final_merged_sha":"931ee687540c9a77bd76ea5c902076e97440d479","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T19:02:36Z","started_at":"2026-10-13T19:02:36Z","completed_at":"2026-10-13T21:24:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T21:37:17.587340489Z","timing_metadata_checked_at":"2026-10-13T21:37:17.587340489Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T21:37:17.590716279Z","region_metadata_checked_at":"2026-10-13T21:37:17.590716279Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"7b9832afe9e5ede4fad84116f35ee2bca0c750ccecf67b335672d92b74614c77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110083918836273152","occurred_at":"2026-10-13T19:02:36Z","failed_at":"alert","signature_id":"6f5244fc0fea589d70c2fa4e58a09e522022dcd0de6fff81e3ffbe8c544084ae","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T20:14:21Z\n Ended: 2026-10-13T20:56:21Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6273152-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j6273152-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}},{"row_id":"f50e45fae0f68984f82d5c1466d232a75f7819749c8a06cbb2ee6a78b4517811","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110083918836273152","occurred_at":"2026-10-13T19:02:36Z","failed_at":"alert","signature_id":"751897ff3429e507ba41ef73f8187f9a0987de78f57b98795b44735681e9a017","test_name":"[aro-hcp-observability] [svc] alert userJourneyFrontendLatency6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T20:38:20Z\n Ended: 2026-10-13T21:03:20Z\n Severity: Sev3\n Labels: alertname=\"userJourneyFrontendLatency6h30m\", cluster=\"ci01-j6273152-svc\", component=\"slo\", long_window=\"6h\", region=\"centralus\", severity=\"3\", short_window=\"30m\", slo=\"frontend-latency\"\n Description: More than 6% of Frontend requests on cluster ci01-j6273152-svc exceeded 1s on both the 6h and 30m windows (6x burn of the 1% latency budget).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b","failure_pattern":"alert [svc] userJourneyFrontendLatency6h30m fired","search_query":"alert [svc] userJourneyFrontendLatency6h30m fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","addfa32c4c8681e954bead4d63f7ee51babad93b3eb924950aeb173e9712942b"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110075452461682688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7469,"pr_state":"closed","pr_sha":"a2d7fd940b1ba037c80cee86cf5b380e84ab4193","final_merged_sha":"a2d7fd940b1ba037c80cee86cf5b380e84ab4193","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T18:01:34Z","started_at":"2026-10-13T18:01:34Z","completed_at":"2026-10-13T20:47:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T21:06:31.50241777Z","timing_metadata_checked_at":"2026-10-13T21:06:31.50241777Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T21:06:31.506927355Z","region_metadata_checked_at":"2026-10-13T21:06:31.506927355Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"4e8e21abddd8e970058598b86f949f98606300be9b71c13cb432d124bbf871b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110075452461682688","occurred_at":"2026-10-13T18:01:34Z","failed_at":"alert","signature_id":"dea3871c0f87e0576e988742c681e43a182dd667db806dd19107b71b0897c1c0","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T20:26:27Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1682688-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.46:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1682688/providers/microsoft.monitor/accounts/services-j1682688\", namespace=\"klusterlet-2te3djdf1hkgonc8k4i71prc94m2s3ap\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qlgkj\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2te3djdf1hkgonc8k4i71prc94m2s3ap on cluster ci01-j1682688-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"6a681ce8819a64ce2b74ca0bc198a2403f7c2a61596a9db41d091c30ea948ad9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110075452461682688","occurred_at":"2026-10-13T18:01:34Z","failed_at":"alert","signature_id":"920e9198dd57a8245eaf4811aaa20836cb9e3378296216a9590cd685a85e0bbf","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T19:33:58Z\n Ended: 2026-10-13T19:41:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1682688-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2te3dejb287gndjkgcapmskarjrlpqqa-m7o3o1o1z6r6k7r\", pod=\"router-859555d867-955d2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2te3dejb287gndjkgcapmskarjrlpqqa-m7o3o1o1z6r6k7r/router-859555d867-955d2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110080447856775168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7495,"pr_state":"open","pr_sha":"3c936e16bdf98e15f0724566b1efeaef8d31a693","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T17:40:39Z","started_at":"2026-10-13T17:40:39Z","completed_at":"2026-10-13T20:59:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T21:16:44.061269472Z","timing_metadata_checked_at":"2026-10-13T21:16:44.061269472Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T21:16:44.064790661Z","region_metadata_checked_at":"2026-10-13T21:16:44.064790661Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"078ce7f0dea7d2a92c9de2555db404b1a46d92c84bf2d1c1865f0b2b2968a2e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110080447856775168","occurred_at":"2026-10-13T17:40:39Z","failed_at":"alert","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j6775168 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T19:48:13Z\n Ended: 2026-10-13T20:21:13Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired"}},{"row_id":"2c0554e1724315b8dd8627374ccca112987fa82902bdba0e720229a6849424a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110080447856775168","occurred_at":"2026-10-13T17:40:39Z","failed_at":"alert","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T20:03:58Z\n Ended: 2026-10-13T20:15:58Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6775168-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"4cd0be6eaf56bbbe706a7387e487aef159b26ebdbd81ceae4d4651d462c02518","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110080447856775168","occurred_at":"2026-10-13T17:40:39Z","failed_at":"alert","signature_id":"3845ce22d709f7d6f88c6b78d29a932df4b7e3435bd9d98ded715b56bb04b04e","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption Burst High - arohcpci01-rp-j6775168 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T19:42:33Z\n Ended: 2026-10-13T20:09:33Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 95% averaged over a 5-minute window, evaluated every minute. This catches short RU-saturation bursts (e.g. mass cluster creation at the start of E2E tests) that exceed provisioned throughput and can lead to 429 throttling before the 10-minute sustained alert fires. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption Burst High - \u003ccosmos-account\u003e fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","9f75b69b94f5e69dd2ea89d590d15c51614ce3eda1965d4efc965f1caf5f8e2b"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7249/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110063387919519744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7249,"pr_state":"closed","pr_sha":"4d2db88e2c3c4e0894b79d6ffabc5a6460dc5ff6","final_merged_sha":"4d2db88e2c3c4e0894b79d6ffabc5a6460dc5ff6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-13T17:40:26Z","started_at":"2026-10-13T17:40:26Z","completed_at":"2026-10-13T20:02:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T20:12:33.113120255Z","timing_metadata_checked_at":"2026-10-13T20:12:33.113120255Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T20:12:33.115687147Z","region_metadata_checked_at":"2026-10-13T20:12:33.115687147Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7477,"pr_state":"closed","pr_sha":"029d181f753f08545d8b2465284790ac0da13db4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T17:32:02Z","started_at":"2026-10-13T17:32:02Z","completed_at":"2026-10-13T20:13:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T20:23:28.135174496Z","timing_metadata_checked_at":"2026-10-13T20:23:28.135174496Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T20:23:28.137866987Z","region_metadata_checked_at":"2026-10-13T20:23:28.137866987Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"08ac899d03e8d5c06ed5983508a62108e2d5f44f12480f6610d5016c319fd708","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","failed_at":"e2e","signature_id":"c15f77333166c293b45237bf13679bc471c7c7da27a687c44335f6bfc722d028","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x194b8fb466e0\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x194b8fb466c0\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x194b8fb466a0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x194b8fb466e0\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x194b8fb466c0\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x194b8fb466a0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0"}},{"row_id":"26618674ff403922f1b386c0f363b5baa9991698cee6be1987e23e99f65a42df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","failed_at":"e2e","signature_id":"f778b2e59ac991a933120117b78a9c24dc834541831bf4f98f55f52a6dba1078","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2b0867292620\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x2b08695be1d0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x2b0867292620\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x2b08695be1d0\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0"}},{"row_id":"93cdc8048cbd32a848a879f846aed964b6b372d2a181d55e9bba888d0aa40f64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","failed_at":"e2e","signature_id":"2ff884e11f43410ad920aa9b4baf8cb219805fce930bc1a46992b3fb5753fbd3","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:134]: failed to resolve configured channel candidate-5.1\nUnexpected error:\n \u003c*fmt.wrapError | 0xda3675c58e0\u003e: \n after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\n {\n msg: \"after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n err: \u003c*errors.errorString | 0xda366e3dab0\u003e{\n s: \"1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:134]: failed to resolve configured channel candidate-5.1\nUnexpected error:\n \u003c*fmt.wrapError | 0xda3675c58e0\u003e: \n after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\n {\n msg: \"after 4 attempts: 1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n err: \u003c*errors.errorString | 0xda366e3dab0\u003e{\n s: \"1 releases found in https://api.openshift.com/api/upgrades_info/graph?arch=multi\u0026channel=candidate-5.1, which is not enough for the requested 1 offset\",\n },\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5207ee83cecbb4a62f10c9755387275f2a4075a267928e8754031a9cdbdeb408","failure_pattern":"after 4 attempts: 1 releases found in \u003curl\u003e which is not enough for the requested 1 offset","search_query":"occurred"}},{"row_id":"a35795eca1ee3699eb1c8d91998b5bb735cc95314a93beca90b3f1aa94c73664","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","failed_at":"e2e","signature_id":"6334833f41bf18b21c6f6a2861528f92d6fb485ca2f0d52c7d87545a02b6cbc9","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x14a7efc6800\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x14a7fffde80\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x14a7efc6800\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.002s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x14a7fffde80\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0"}},{"row_id":"ab932341c6c392d79a3a9210229a477520e538cade4bea11874db032c3bb2d9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","failed_at":"e2e","signature_id":"be104b66a02409cc4adfb532056ddabf4b40d0214823d8003994a6056e0fd4a4","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-c5gl2m96t572/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-c5gl2m96t572/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-c5gl2m96t572/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"bfe40aeab8d9f11178b3bfe8088969685a06f005005e423dbfa17d548d0df62a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","failed_at":"e2e","signature_id":"5c557e1356eb3b63a8003453927c70609ed1a2e90406457fdd54fc88e1fb53e5","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1b7c5f7105e0\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x1b7c6068e900\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:522]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1b7c5f7105e0\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.005s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n err: \u003c*errors.errorString | 0x1b7c6068e900\u003e{\n s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0"}},{"row_id":"d736483e099c7f22555896abc5121f1663f0b45edf423ed57acecefd6c2db3a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","failed_at":"e2e","signature_id":"7728ea8ada183ffb061ff6026d3b0066e425d012b33ecc69f14c383fb512405c","test_name":"ARO-HCP should serve the default ingress certificate and allow a customer ingress certificate for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:405]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xabc2b9fef0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.11:42173-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.11:42173-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert.go:405]: Timed out after 300.005s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xabc2b9fef0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.11:42173-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.129.0.11:42173-\u003e172.30.0.10:53: read: connection refused\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}},{"row_id":"e940760a7236f49f5ef79cf7dc1f7e8584d42fa9395532cce15d6002d1aa41cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110068844331536384","occurred_at":"2026-10-13T17:32:02Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T18:58:36Z\n Ended: 2026-10-13T19:18:32Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1536384-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","5207ee83cecbb4a62f10c9755387275f2a4075a267928e8754031a9cdbdeb408","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110066955829383168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7463,"pr_state":"open","pr_sha":"5c08ce538ef6e2f295a6e0908946505526d879e0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T17:24:50Z","started_at":"2026-10-13T17:24:50Z","completed_at":"2026-10-13T20:09:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T20:23:28.007462794Z","timing_metadata_checked_at":"2026-10-13T20:23:28.007462794Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T20:23:28.011240982Z","region_metadata_checked_at":"2026-10-13T20:23:28.011240982Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7483/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110050590087188480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7483,"pr_state":"open","pr_sha":"a89bdefc7a342554bd430f08f7b5efd3ed297494","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T16:50:38Z","started_at":"2026-10-13T16:50:38Z","completed_at":"2026-10-13T19:56:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T20:12:33.1305319Z","timing_metadata_checked_at":"2026-10-13T20:12:33.1305319Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T20:12:33.133091492Z","region_metadata_checked_at":"2026-10-13T20:12:33.133091492Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7248,"pr_state":"closed","pr_sha":"29ee7d9651963ed9c17688c509801cba7771db4c","final_merged_sha":"29ee7d9651963ed9c17688c509801cba7771db4c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T16:43:38Z","started_at":"2026-10-13T16:43:38Z","completed_at":"2026-10-13T19:58:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T20:12:32.674226943Z","timing_metadata_checked_at":"2026-10-13T20:12:32.674226943Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T20:12:32.677417933Z","region_metadata_checked_at":"2026-10-13T20:12:32.677417933Z"},"failed_at":["alert","e2e"],"failed_test_count":12,"occurrences":[{"row_id":"2540235d01f630ed32cd2f06731967293d21773839939d164b7fb236e0ee84e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"e2e","signature_id":"be1b7e4b70ae40bb176b235bc31683c57b20ea13e1c6ce838e84dccb47cf9506","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-cjhwhd\" with version 5.0.0-rc.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2dcd6debe660\u003e: \n failed to create HCP cluster cluster-zstream-5-0-cjhwhd: failed waiting for cluster=\"cluster-zstream-5-0-cjhwhd\" in resourcegroup=\"rg-zstream-upgrade-5-0-zbcrdfvtst44\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\",\n \"name\": \"0fa13488-8ecd-48a9-95b8-c369b6566865\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:12.815750049Z\",\n \"endTime\": \"2026-10-13T18:47:19.806209332Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n correlation ID: d259ddfa-e906-4fbb-974b-3c977af82a32\n client request ID: be895625-b480-4da8-a968-9e02baae7cef\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-cjhwhd\" with version 5.0.0-rc.5 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0x2dcd6debe660\u003e: \n failed to create HCP cluster cluster-zstream-5-0-cjhwhd: failed waiting for cluster=\"cluster-zstream-5-0-cjhwhd\" in resourcegroup=\"rg-zstream-upgrade-5-0-zbcrdfvtst44\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\",\n \"name\": \"0fa13488-8ecd-48a9-95b8-c369b6566865\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:12.815750049Z\",\n \"endTime\": \"2026-10-13T18:47:19.806209332Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0fa13488-8ecd-48a9-95b8-c369b6566865\n correlation ID: d259ddfa-e906-4fbb-974b-3c977af82a32\n client request ID: be895625-b480-4da8-a968-9e02baae7cef\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d55a541fe54a611e130419c1f169b8c73c34a953ed35defd84c0a24bbe0a88e3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"39c621be12570d8bf00739dd2edcb5f4deae87ed991a8cf9c23553460e6be137","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"alert","signature_id":"cd835111d6723302e38161708c21c4a3215a58c651e4b372392909f248560622","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 14 time(s) (4 unknown, 10 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T18:27:25Z\n Ended: 2026-10-13T19:00:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T18:28:35Z\n Ended: 2026-10-13T19:00:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-13T18:28:36Z\n Ended: 2026-10-13T19:00:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-13T18:28:36Z\n Ended: 2026-10-13T19:00:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r/etcd-2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"40b052c9976d7cb8ea741981014a860a19f9b21feada6244c9bc2ec781cfa457","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"e2e","signature_id":"afbbb0fd16401b97b4a184026640540584d09079290abbba4ab2babc53541b3a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4x5sn82fcphp/cluster-candidate-4-22-zrq2s6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x19aa63530420\u003e: \n failed to create HCP cluster cluster-candidate-4-22-zrq2s6: failed waiting for cluster=\"cluster-candidate-4-22-zrq2s6\" in resourcegroup=\"rg-candidate-4-22-4x5sn82fcphp\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a9c2b23d-2530-4218-9438-90a72b3ed5c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a9c2b23d-2530-4218-9438-90a72b3ed5c3\",\n \"name\": \"a9c2b23d-2530-4218-9438-90a72b3ed5c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:33.062727378Z\",\n \"endTime\": \"2026-10-13T18:47:39.762455137Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a9c2b23d-2530-4218-9438-90a72b3ed5c3\n correlation ID: 0a0f8159-fd50-4585-9f5a-4fe4a3b75abe\n client request ID: 2d0d801d-1f09-4e1e-b3d0-c47d56f382dc\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-4x5sn82fcphp/cluster-candidate-4-22-zrq2s6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x19aa63530420\u003e: \n failed to create HCP cluster cluster-candidate-4-22-zrq2s6: failed waiting for cluster=\"cluster-candidate-4-22-zrq2s6\" in resourcegroup=\"rg-candidate-4-22-4x5sn82fcphp\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a9c2b23d-2530-4218-9438-90a72b3ed5c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a9c2b23d-2530-4218-9438-90a72b3ed5c3\",\n \"name\": \"a9c2b23d-2530-4218-9438-90a72b3ed5c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:33.062727378Z\",\n \"endTime\": \"2026-10-13T18:47:39.762455137Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a9c2b23d-2530-4218-9438-90a72b3ed5c3\n correlation ID: 0a0f8159-fd50-4585-9f5a-4fe4a3b75abe\n client request ID: 2d0d801d-1f09-4e1e-b3d0-c47d56f382dc\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4445b0396bd0f9d4ae827f226f370ae16206475f3c0fd60904c710397c84d98c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"e2e","signature_id":"48f96c794b4248e434313e5383dab85570d91b004b93d8173274ca3b89e3bd5b","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-qcrtxl\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x124373e0e100\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-qcrtxl: failed waiting for cluster=\"cp-ystream-upgrade-4-23-qcrtxl\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-78kjmz9gspg6\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d1a5e31e-a45c-4250-99d1-c48ac813ea28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d1a5e31e-a45c-4250-99d1-c48ac813ea28\",\n \"name\": \"d1a5e31e-a45c-4250-99d1-c48ac813ea28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:30.082143101Z\",\n \"endTime\": \"2026-10-13T18:47:39.605099419Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d1a5e31e-a45c-4250-99d1-c48ac813ea28\n correlation ID: 361fa06a-5b9c-42df-b411-6ad9494cda34\n client request ID: 5835d195-02fb-460c-b9e4-3f1748f27ec9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-qcrtxl\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0x124373e0e100\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-qcrtxl: failed waiting for cluster=\"cp-ystream-upgrade-4-23-qcrtxl\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-78kjmz9gspg6\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d1a5e31e-a45c-4250-99d1-c48ac813ea28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d1a5e31e-a45c-4250-99d1-c48ac813ea28\",\n \"name\": \"d1a5e31e-a45c-4250-99d1-c48ac813ea28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:30.082143101Z\",\n \"endTime\": \"2026-10-13T18:47:39.605099419Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d1a5e31e-a45c-4250-99d1-c48ac813ea28\n correlation ID: 361fa06a-5b9c-42df-b411-6ad9494cda34\n client request ID: 5835d195-02fb-460c-b9e4-3f1748f27ec9\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4ebd149bf2e8b3395b7f27d30975cbc756d2132c3fa41af7a2f7cc315f6f8f12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"alert","signature_id":"1adc25e471ad62d434570973bdec6d11ffd2186cddd280c054d7dedcef620819","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T18:51:30Z\n Ended: 2026-10-13T19:06:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T18:52:27Z\n Ended: 2026-10-13T19:07:25Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-13T18:52:27Z\n Ended: 2026-10-13T19:06:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-13T18:53:28Z\n Ended: 2026-10-13T19:05:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-13T18:53:28Z\n Ended: 2026-10-13T19:19:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"6f4c9485d5fb0b2e4c6868ef5670f43b75be665fa8be5a71e9bfe5d9ed0d07b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"e2e","signature_id":"b136a9b186b73309fc330d90067542e4a0acbb1b86014bb556a7bc37a5f5c48c","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x384da2442800\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-65x7ldfvpfrb\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n \"name\": \"8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:19:32.972571872Z\",\n \"endTime\": \"2026-10-13T18:48:39.815141519Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n correlation ID: 7e9c28a1-2d97-4241-837c-65dbd7be5ebd\n client request ID: 76ea1b85-c87c-48c0-b8b1-76222a78e914\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0x384da2442800\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-65x7ldfvpfrb\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n \"name\": \"8d7260b8-3447-49f3-b1d1-85b028cd4875\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:19:32.972571872Z\",\n \"endTime\": \"2026-10-13T18:48:39.815141519Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d7260b8-3447-49f3-b1d1-85b028cd4875\n correlation ID: 7e9c28a1-2d97-4241-837c-65dbd7be5ebd\n client request ID: 76ea1b85-c87c-48c0-b8b1-76222a78e914\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"727b90d738458fae42fd743aac06537d1157ba87ee4430dc05269762a9e0b6d7","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"77f6f92c95c7a6a723927890e2a1186cfbfea7ee711ea4a4f915ae04d3483ae3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"e2e","signature_id":"f76417099900daa68e51ebde43fef8462ff51d9cfbe2e05ce62c4518ce93529f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-dllvsm\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x28758e830200\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-dllvsm: failed waiting for cluster=\"cp-ystream-upgrade-4-21-dllvsm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-kqbmkzfdlvpv\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\",\n \"name\": \"96ec3c05-7e89-48a9-8588-f769d1d82955\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:30.340236179Z\",\n \"endTime\": \"2026-10-13T18:47:39.7699444Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n correlation ID: 8e81f893-e896-41de-bfb5-ecaf7ea09f96\n client request ID: 8d55acf1-8d30-4678-b2f6-7eed037af357\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-dllvsm\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0x28758e830200\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-dllvsm: failed waiting for cluster=\"cp-ystream-upgrade-4-21-dllvsm\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-kqbmkzfdlvpv\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\",\n \"name\": \"96ec3c05-7e89-48a9-8588-f769d1d82955\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:30.340236179Z\",\n \"endTime\": \"2026-10-13T18:47:39.7699444Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/96ec3c05-7e89-48a9-8588-f769d1d82955\n correlation ID: 8e81f893-e896-41de-bfb5-ecaf7ea09f96\n client request ID: 8d55acf1-8d30-4678-b2f6-7eed037af357\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d55a541fe54a611e130419c1f169b8c73c34a953ed35defd84c0a24bbe0a88e3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a5f317004946357737c5d26569a45f0f2f8a5941a116fb09502500ebc8fda8d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"alert","signature_id":"e9d0a6cb09c8c2cfd53af2a6634e712d86d6ad07e23aa28dffb4ebf4cfd08256","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T18:37:29Z\n Ended: 2026-10-13T18:55:26Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T18:37:29Z\n Ended: 2026-10-13T18:54:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2g0ue1d1l7frlbcjrgu6374q9f127-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-13T18:38:27Z\n Ended: 2026-10-13T18:55:26Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-13T18:38:27Z\n Ended: 2026-10-13T18:55:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-13T18:38:27Z\n Ended: 2026-10-13T18:54:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-13T18:38:28Z\n Ended: 2026-10-13T18:54:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2fuhn0rfvdiv69jgohnlv63fgjjka-q9h0p7f7k8t5e0e/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-13T18:38:28Z\n Ended: 2026-10-13T18:55:26Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2ftrfr2171c9vgv1tegaimecvl6vj-g6a2f9w2l2r4x0i/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-13T18:38:28Z\n Ended: 2026-10-13T18:54:27Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/hcps-j2423040\", namespace=\"ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te2fpcp41do03q2ch5ajhtin6o2onk6-n6k1o6x0q6g7e9r/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"b599e86ec39d8ef3e9a8cb9f1f2e8fe945c837112de0451042ed12126a55a1ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"alert","signature_id":"ff049ca30612ae3c5d981f61312a933989a2df4db1c20adc6d1b9ff278e63fd3","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T18:51:01Z\n Ended: 2026-10-13T18:58:00Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T18:51:02Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-13T18:51:02Z\n Ended: 2026-10-13T18:59:03Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2fpcp41do03q2ch5ajhtin6o2onk6\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2fpcp41do03q2ch5ajhtin6o2onk6/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-13T18:51:02Z\n Ended: 2026-10-13T18:58:00Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-13T18:52:01Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-13T18:52:01Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-13T18:52:01Z\n Ended: 2026-10-13T18:58:01Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-13T18:52:01Z\n Ended: 2026-10-13T18:58:01Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2ftrfr2171c9vgv1tegaimecvl6vj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-13T18:52:01Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2g0ue1d1l7frlbcjrgu6374q9f127/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-13T18:52:58Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-13T18:52:58Z\n Ended: 2026-10-13T18:59:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2fuhn0rfvdiv69jgohnlv63fgjjka/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-13T18:54:01Z\n Ended: 2026-10-13T19:01:00Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2423040-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.82:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2423040/providers/microsoft.monitor/accounts/services-j2423040\", namespace=\"klusterlet-2te2gdnt0p29rhn6u0lod1kmga0s4ejn\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mgp99\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2te2gdnt0p29rhn6u0lod1kmga0s4ejn/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"ba102b5b1d614625280a7f1b5788410dd4aefaaa36d1697df0d62d3b1083ac94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"e2e","signature_id":"077f299c2a7977d35eb0dc0f0f3806d25b08b958ea3c8d1b9f28717ecf19fa10","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:125]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x178e97a22840\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-w76dhl6mtjzd\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\",\n \"name\": \"235e4bf2-54cf-43d0-ab9b-65d9a733710d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:43.086930646Z\",\n \"endTime\": \"2026-10-13T18:47:49.831018491Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\n correlation ID: 077ff34a-ea54-4d78-afd3-f4054328a8bd\n client request ID: 612adb31-4506-4736-9005-28fa95bb228f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:125]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x178e97a22840\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-w76dhl6mtjzd\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\",\n \"name\": \"235e4bf2-54cf-43d0-ab9b-65d9a733710d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:43.086930646Z\",\n \"endTime\": \"2026-10-13T18:47:49.831018491Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/235e4bf2-54cf-43d0-ab9b-65d9a733710d\n correlation ID: 077ff34a-ea54-4d78-afd3-f4054328a8bd\n client request ID: 612adb31-4506-4736-9005-28fa95bb228f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bd04eacb6a49cd7f2f36e8503216e9deaa12258302da9be5a806ac27fcaa0f5a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"e2e","signature_id":"afb4e7bf5663c4264337c9d1e677c0444a5f93e3e544da615e860a878027556d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-pgbspldcrvd7/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-pgbspldcrvd7/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-c5gl2m96t572/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"e794bdde2b6653209ee51f77541a27cce8adbd66a2f5e45de0261048ca19f29f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7248/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110048828542423040","occurred_at":"2026-10-13T16:43:38Z","failed_at":"e2e","signature_id":"61bc9fb2baad09f02ca746a494c19b296e155e5562e460538f3b5c55bfa4f3f6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-mcx8fpbfcpnk/cluster-candidate-5-0-vlgchl should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x109506636520\u003e: \n failed to create HCP cluster cluster-candidate-5-0-vlgchl: failed waiting for cluster=\"cluster-candidate-5-0-vlgchl\" in resourcegroup=\"rg-candidate-5-0-mcx8fpbfcpnk\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1b967a2-e91b-49c3-83e2-d3b8a97a8f24\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1b967a2-e91b-49c3-83e2-d3b8a97a8f24\",\n \"name\": \"a1b967a2-e91b-49c3-83e2-d3b8a97a8f24\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:32.401074593Z\",\n \"endTime\": \"2026-10-13T18:47:39.748542315Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1b967a2-e91b-49c3-83e2-d3b8a97a8f24\n correlation ID: 8db3f564-6943-4e86-ae53-344e19b24905\n client request ID: 395fee37-68bf-4f23-8767-6c77bcf873de\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-mcx8fpbfcpnk/cluster-candidate-5-0-vlgchl should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x109506636520\u003e: \n failed to create HCP cluster cluster-candidate-5-0-vlgchl: failed waiting for cluster=\"cluster-candidate-5-0-vlgchl\" in resourcegroup=\"rg-candidate-5-0-mcx8fpbfcpnk\" to finish creating: GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1b967a2-e91b-49c3-83e2-d3b8a97a8f24\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1b967a2-e91b-49c3-83e2-d3b8a97a8f24\",\n \"name\": \"a1b967a2-e91b-49c3-83e2-d3b8a97a8f24\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T18:18:32.401074593Z\",\n \"endTime\": \"2026-10-13T18:47:39.748542315Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2423040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a1b967a2-e91b-49c3-83e2-d3b8a97a8f24\n correlation ID: 8db3f564-6943-4e86-ae53-344e19b24905\n client request ID: 395fee37-68bf-4f23-8767-6c77bcf873de\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","727b90d738458fae42fd743aac06537d1157ba87ee4430dc05269762a9e0b6d7","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","d55a541fe54a611e130419c1f169b8c73c34a953ed35defd84c0a24bbe0a88e3"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110045082655657984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7477,"pr_state":"closed","pr_sha":"c8eaee8afc8ecc8c3ca6bbb23d5630f981cf07cb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T16:28:45Z","started_at":"2026-10-13T16:28:45Z","completed_at":"2026-10-13T16:44:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T16:52:38.547341577Z","timing_metadata_checked_at":"2026-10-13T16:52:38.547341577Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-13T16:52:38.550784866Z","region_metadata_checked_at":"2026-10-13T17:56:19.42097468Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8e5232f3cce7f7965348b7bd0df3be15e0fbf223795977f78d76a69753c645b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110045082655657984","occurred_at":"2026-10-13T16:28:45Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110043835399671808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7504,"pr_state":"open","pr_sha":"c0e2f18ace51d06f9bcf47af05e7fa0bec608ae0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T16:23:47Z","started_at":"2026-10-13T16:23:47Z","completed_at":"2026-10-13T19:36:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T19:51:48.092476818Z","timing_metadata_checked_at":"2026-10-13T19:51:48.092476818Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T19:51:48.095548908Z","region_metadata_checked_at":"2026-10-13T19:51:48.095548908Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"d105a739519a12d4f7eaf8a0b10d01384cf0e57619365b02129bfc4e805e38c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7504/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110043835399671808","occurred_at":"2026-10-13T16:23:47Z","failed_at":"alert","signature_id":"75b1290532c2ce606ea20bb999b66cb1911da401025d28a68dcef6b1a4ebb765","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T19:14:04Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9671808-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.101:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9671808/providers/microsoft.monitor/accounts/services-j9671808\", namespace=\"klusterlet-2te2bel6rgtolfhffo3974figin61sko\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z7lt7\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2te2bel6rgtolfhffo3974figin61sko on cluster ci01-j9671808-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7502,"pr_state":"closed","pr_sha":"46b8ad997c74f398f17e0d97e729473b9c3cfbc6","final_merged_sha":"4e388171c7cac99a0d73a3ae18b94e46f3e3866f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T16:19:32Z","started_at":"2026-10-13T16:19:32Z","completed_at":"2026-10-13T18:47:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:58:26.459872123Z","timing_metadata_checked_at":"2026-10-13T18:58:26.459872123Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:58:26.463187713Z","region_metadata_checked_at":"2026-10-13T18:58:26.463187713Z"},"failed_at":["alert","provision"],"failed_test_count":4,"occurrences":[{"row_id":"0146cd3901cda00f2465c7cc447a5b37792017079c1eed951e20eaee83f8bd72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","failed_at":"alert","signature_id":"26ad01559a977d2ccbf9de49f0e0a12ad85e844d43c5c076830f46ba6268215f","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T18:24:20Z\n Ended: 2026-10-13T18:31:19Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j4029440-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T18:24:20Z\n Ended: 2026-10-13T18:31:19Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j4029440-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}},{"row_id":"36b6fd3304707dda86f074225d50b77c938773fcfe0e29c485d61af12029fc51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","failed_at":"alert","signature_id":"61b46f21ba4b01a9a3f1d792838519213b86b988924028497bb2806fe1283487","test_name":"[aro-hcp-observability] [svc] alert MaestroServerNoReadyReplicas does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T18:40:06Z\n Severity: Sev2\n Labels: alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j4029440-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4029440/providers/microsoft.monitor/accounts/services-j4029440\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-k5lj9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-13T18:40:06Z\n Severity: Sev2\n Labels: alertname=\"MaestroServerNoReadyReplicas\", cluster=\"ci01-j4029440-svc\", component=\"maestro\", container=\"kube-state-metrics\", deployment=\"maestro\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.59:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4029440/providers/microsoft.monitor/accounts/services-j4029440\", namespace=\"maestro\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-k5lj9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"critical\"\n Description: No maestro-server replicas have been Ready for 10 minutes. Readiness is gated on Postgres connectivity, so maestro is likely unable to reach its database; Clusters Service will get 503 and HCP cluster provisioning will stall.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c35e869ec847e18ffe6465660e3d583636fa828b2ff0c004e57c6941d2b85e2e","failure_pattern":"alert [svc] MaestroServerNoReadyReplicas fired","search_query":"alert [svc] MaestroServerNoReadyReplicas fired"}},{"row_id":"e9709390ff94a5f197294e3215c1402ffd48413c59e5abcd94f53676d7bada76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","failed_at":"alert","signature_id":"761776c84216b3b60fba1a6f060188d98d184222bdf626e7b5bac2005be4c5d7","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 17 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-hllvl\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-hllvl has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14034537-vmss000000-xc7fq\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-14034537-vmss000000-xc7fq has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-4k67c\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-4k67c has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-czf22\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-czf22 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14034537-vmss000002-9swqr\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-14034537-vmss000002-9swqr has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"hubble-relay-58594b5dd7-nnl6j\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/hubble-relay-58594b5dd7-nnl6j has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-14034537-vmss000001-8tbkl\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/eraser-aks-system-14034537-vmss000001-8tbkl has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-jvbz8\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-jvbz8 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-nn274\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-nn274 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-13T18:32:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-7rvmk\", region=\"westus3\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-7rvmk has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-c67c4f659-qcdm6\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-c67c4f659-qcdm6 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-7dbd665f4c-8rntv\", region=\"westus3\", severity=\"warning\"\n Description: Pod maestro/maestro-7dbd665f4c-8rntv has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-c67c4f659-c7v65\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-c67c4f659-c7v65 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"maestro\", pod=\"maestro-7dbd665f4c-474rm\", region=\"westus3\", severity=\"warning\"\n Description: Pod maestro/maestro-7dbd665f4c-474rm has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-c67c4f659-4zmsg\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-c67c4f659-4zmsg has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-698d94bbf-pwqx5\", region=\"westus3\", severity=\"warning\"\n Description: Pod sessiongate/sessiongate-698d94bbf-pwqx5 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-10-13T18:34:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4029440-svc\", component=\"kubernetes-infrastructure\", namespace=\"sessiongate\", pod=\"sessiongate-698d94bbf-7crpb\", region=\"westus3\", severity=\"warning\"\n Description: Pod sessiongate/sessiongate-698d94bbf-7crpb has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"f9d7885f1fb99bf31706084151bfb4f2d0fbc77fecf49b0f62ab9b4fbba57397","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7502/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110042774484029440","occurred_at":"2026-10-13T16:19:32Z","failed_at":"provision","signature_id":"2a551764d2de0ec2ce5493d79b0c6e2957d9ec9ebccce1e0772fad9dff29448f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/service/deploy","test_suite":"templatize-pipeline","failure_text":"time=2026-10-13T18:29:04.438Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-13T18:29:04.984Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy values=\"map[azure:map[clientId:e73242a2-1d48-4914-b494-303251483672 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] backend:map[namespace:aro-hcp serviceAccount:backend] broker:map[host:arohcp-ci01-maestro-j4029440.westus3-1.ts.eventgrid.azure.net port:8883] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:maestro-server-j4029440] database:map[azureDb:false containerizedDbImage:docker.io/library/postgres:14.2 containerizedDbPvcCapacity:512Mi debug:false host:maestro-db maxOpenConnections:50 name:maestro password:TheBlurstOfTimes port:5432 user:maestro-server] deployment:map[replicas:2 resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]] zoneCount:3] fleet:map[namespace:fleet serviceAccount:fleet] image:map[digest:sha256:ccc6c4381bbbc6a7af65d4c3886e20baea76f7bb95ea6048b33ee31e8046e5cb registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] istio:map[restrictIngress:true] maestro:map[glog_v:2 grpcBindPort:8090 healthCheckBindPort:8083 httpBindPort:8000 httpReadTimeout:60s httpWriteTimeout:60s labelMetricsInclusionDuration:168h serviceAccount:maestro] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro:pull workloadIdentityClientId:1176baba-6cca-46e9-86bb-3a9c587b6d0c workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-10-13T18:29:04.984Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-13T18:29:05.371Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy namespace=maestro\ntime=2026-10-13T18:29:05.621Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy dryRun=false\ntime=2026-10-13T18:29:05.621Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-13T18:29:05.751Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy\ntime=2026-10-13T18:39:13.113Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded\"\ntime=2026-10-13T18:39:13.113Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9edb2d39bdff9cf1586610e7e1778893f0414c6b0749314791e92df31b54ec6f","failure_pattern":"failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro not ready. status: Failed, message: Progress deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","9edb2d39bdff9cf1586610e7e1778893f0414c6b0749314791e92df31b54ec6f","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","c35e869ec847e18ffe6465660e3d583636fa828b2ff0c004e57c6941d2b85e2e"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110039601807626240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7397,"pr_state":"closed","pr_sha":"88f953df45eba2a91a51c0ad97acee3993bb57a9","final_merged_sha":"88f953df45eba2a91a51c0ad97acee3993bb57a9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-13T16:06:58Z","started_at":"2026-10-13T16:06:58Z","completed_at":"2026-10-13T19:09:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T19:19:25.019870648Z","timing_metadata_checked_at":"2026-10-13T19:19:25.019870648Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T19:19:25.025377032Z","region_metadata_checked_at":"2026-10-13T19:19:25.025377032Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110039512951296000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7488,"pr_state":"closed","pr_sha":"e343803f365d228cc3e0070ed414c588a26b5e9f","final_merged_sha":"e343803f365d228cc3e0070ed414c588a26b5e9f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T16:06:36Z","started_at":"2026-10-13T16:06:36Z","completed_at":"2026-10-13T19:53:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T20:12:33.156279919Z","timing_metadata_checked_at":"2026-10-13T20:12:33.156279919Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T20:12:33.15885311Z","region_metadata_checked_at":"2026-10-13T20:12:33.15885311Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"3615a17c25661f537a114ec85c4d51b8cee0c7cb83d95d707d62ae6e8315f6e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110039512951296000","occurred_at":"2026-10-13T16:06:36Z","failed_at":"alert","signature_id":"c8d1bc3849a91c7150052cbd8fd01e8bacd6acfee15dec3a5b9f1ee2a8d69a1f","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T19:29:44Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j1296000-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1296000/providers/microsoft.monitor/accounts/services-j1296000\", namespace=\"klusterlet-2te2i26cvbt70m1icpl3fmd77s52ciq8\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-86rpv\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2te2i26cvbt70m1icpl3fmd77s52ciq8 on cluster ci01-j1296000-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110039423042195456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7162,"pr_state":"closed","pr_sha":"be72cb34574df43544fc0052fd929a88e795ba1b","final_merged_sha":"bf3311f63a37396e26e3b7417a174b8fca34b285","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T16:05:54Z","started_at":"2026-10-13T16:05:54Z","completed_at":"2026-10-13T19:24:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T19:40:58.112776498Z","timing_metadata_checked_at":"2026-10-13T19:40:58.112776498Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T19:40:58.116149888Z","region_metadata_checked_at":"2026-10-13T19:40:58.116149888Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"345edadc9c6317ce5f628c26e317afbe835f9d4427f1272ffb728b7bdcbf07d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110039423042195456","occurred_at":"2026-10-13T16:05:54Z","failed_at":"alert","signature_id":"e0aece362630a726f3a415cc6e772ac06ec61590d1f6668bb57f6d646bc54566","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T18:37:57Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j2195456-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.244:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2195456/providers/microsoft.monitor/accounts/services-j2195456\", namespace=\"klusterlet-2te22fpnsean3o8qbccv619h2micg5vv\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-hcksn\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2te22fpnsean3o8qbccv619h2micg5vv on cluster ci01-j2195456-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035934396289024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7484,"pr_state":"closed","pr_sha":"1857413cdb7ec426b141684f303bba8c60341534","final_merged_sha":"1857413cdb7ec426b141684f303bba8c60341534","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-13T15:52:23Z","started_at":"2026-10-13T15:52:23Z","completed_at":"2026-10-13T18:18:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:26:33.516360603Z","timing_metadata_checked_at":"2026-10-13T18:26:33.516360603Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:26:33.519025494Z","region_metadata_checked_at":"2026-10-13T18:26:33.519025494Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7494/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035877961928704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7494,"pr_state":"closed","pr_sha":"94d8c39d819db7835203371dc3ae43dbd81d32e9","final_merged_sha":"94d8c39d819db7835203371dc3ae43dbd81d32e9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-13T15:52:10Z","started_at":"2026-10-13T15:52:10Z","completed_at":"2026-10-13T18:08:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:26:33.801232992Z","timing_metadata_checked_at":"2026-10-13T18:26:33.801232992Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:26:33.804268182Z","region_metadata_checked_at":"2026-10-13T18:26:33.804268182Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035842520059904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7313,"pr_state":"closed","pr_sha":"02b7511e30cc16674b6938c06b7b2aec2ccdcecf","final_merged_sha":"02b7511e30cc16674b6938c06b7b2aec2ccdcecf","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T15:52:02Z","started_at":"2026-10-13T15:52:02Z","completed_at":"2026-10-13T18:14:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:26:33.333682487Z","timing_metadata_checked_at":"2026-10-13T18:26:33.333682487Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:26:33.337045076Z","region_metadata_checked_at":"2026-10-13T18:26:33.337045076Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"95e7119ef9ac7e75817aa8f4043bbe1a7be56549388f0406af34f018044f6b25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035842520059904","occurred_at":"2026-10-13T15:52:02Z","failed_at":"e2e","signature_id":"04454a362557e528929a3344b4548e2c84ce0874d9f75e7bb99b09f4d24248b9","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1261114a4180\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-sd447nrw2b6l\" to finish creating: GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7256f4ec-b394-462d-a65c-c0c44cf424ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7256f4ec-b394-462d-a65c-c0c44cf424ed\",\n \"name\": \"7256f4ec-b394-462d-a65c-c0c44cf424ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:58:45.675244209Z\",\n \"endTime\": \"2026-10-13T17:27:48.586418683Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0z7z4b2j5v2y3n-np-delete-main-bbsn2-mr9vl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-sd447nrw2b6l--managed/providers/Microsoft.Compute/virtualMachines/o0z7z4b2j5v2y3n-np-delete-main-bbsn2-mr9vl to exist\\nMachine o0z7z4b2j5v2y3n-np-delete-main-bbsn2-ms6dr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-sd447nrw2b6l--managed/providers/Microsoft.Compute/virtualMachines/o0z7z4b2j5v2y3n-np-delete-main-bbsn2-ms6dr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0z7z4b2j5v2y3n-np-delete-main-bbsn2-mr9vl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-sd447nrw2b6l--managed/providers/Microsoft.Compute/virtualMachines/o0z7z4b2j5v2y3n-np-delete-main-bbsn2-mr9vl to exist\\nMachine o0z7z4b2j5v2y3n-np-delete-main-bbsn2-ms6dr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-sd447nrw2b6l--managed/providers/Microsoft.Compute/virtualMachines/o0z7z4b2j5v2y3n-np-delete-main-bbsn2-ms6dr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7256f4ec-b394-462d-a65c-c0c44cf424ed\n correlation ID: fc291250-1808-46ce-ae93-ea402484d914\n client request ID: 0188fcc0-06aa-4fd8-81c3-b6ebc6f67822\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1261114a4180\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-sd447nrw2b6l\" to finish creating: GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7256f4ec-b394-462d-a65c-c0c44cf424ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7256f4ec-b394-462d-a65c-c0c44cf424ed\",\n \"name\": \"7256f4ec-b394-462d-a65c-c0c44cf424ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:58:45.675244209Z\",\n \"endTime\": \"2026-10-13T17:27:48.586418683Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0z7z4b2j5v2y3n-np-delete-main-bbsn2-mr9vl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-sd447nrw2b6l--managed/providers/Microsoft.Compute/virtualMachines/o0z7z4b2j5v2y3n-np-delete-main-bbsn2-mr9vl to exist\\nMachine o0z7z4b2j5v2y3n-np-delete-main-bbsn2-ms6dr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-sd447nrw2b6l--managed/providers/Microsoft.Compute/virtualMachines/o0z7z4b2j5v2y3n-np-delete-main-bbsn2-ms6dr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0z7z4b2j5v2y3n-np-delete-main-bbsn2-mr9vl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-sd447nrw2b6l--managed/providers/Microsoft.Compute/virtualMachines/o0z7z4b2j5v2y3n-np-delete-main-bbsn2-mr9vl to exist\\nMachine o0z7z4b2j5v2y3n-np-delete-main-bbsn2-ms6dr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-sd447nrw2b6l--managed/providers/Microsoft.Compute/virtualMachines/o0z7z4b2j5v2y3n-np-delete-main-bbsn2-ms6dr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7256f4ec-b394-462d-a65c-c0c44cf424ed\n correlation ID: fc291250-1808-46ce-ae93-ea402484d914\n client request ID: 0188fcc0-06aa-4fd8-81c3-b6ebc6f67822","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dpn7kns9h5s6\" to finish creating: GET https://rp"}},{"row_id":"a48a5c3be3a391aa8336fffda4b44a92e4be8e2bd40cbe3b6856e93ae37e9e26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035842520059904","occurred_at":"2026-10-13T15:52:02Z","failed_at":"e2e","signature_id":"48a02a7f69908511527f6ebb16b479024cc99ba313ab90e8e3951306e211831f","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x37d94a08f940\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-j6jbclq8czt4\" to finish creating: GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\",\n \"name\": \"f09b4f7f-dd14-4709-a93e-1fc185bebf4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T17:16:34.496076904Z\",\n \"endTime\": \"2026-10-13T17:45:42.970608528Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2 to exist\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2 to exist\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\n correlation ID: fa9a1781-c6d1-4176-bdee-881a0f4c4796\n client request ID: 94e1b016-c599-42c0-a81c-eb85c23ce070\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x37d94a08f940\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-j6jbclq8czt4\" to finish creating: GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\",\n \"name\": \"f09b4f7f-dd14-4709-a93e-1fc185bebf4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T17:16:34.496076904Z\",\n \"endTime\": \"2026-10-13T17:45:42.970608528Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2 to exist\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-67mf2 to exist\\nMachine r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-j6jbclq8czt4--managed/providers/Microsoft.Compute/virtualMachines/r8o8t5e4i0j8y1g-np-1-mc4v9-kxdpm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0059904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f09b4f7f-dd14-4709-a93e-1fc185bebf4a\n correlation ID: fa9a1781-c6d1-4176-bdee-881a0f4c4796\n client request ID: 94e1b016-c599-42c0-a81c-eb85c23ce070\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035013809475584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7489,"pr_state":"closed","pr_sha":"0432acc9844c3e10bd928e071703f37dc5201b10","final_merged_sha":"a800194beb9b2fac67229770cb4864563f1e4d17","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T15:48:44Z","started_at":"2026-10-13T15:48:44Z","completed_at":"2026-10-13T18:08:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:26:33.678896083Z","timing_metadata_checked_at":"2026-10-13T18:26:33.678896083Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:26:33.681423675Z","region_metadata_checked_at":"2026-10-13T18:26:33.681423675Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"102e9a997f65b16ffebe2d70640e72e78cb693d85aa7afbea56d3bf2c84be64e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035013809475584","occurred_at":"2026-10-13T15:48:44Z","failed_at":"e2e","signature_id":"ed688746e174fb91b628587af9f6d4fc2c44f7a9d83bdae64d4e89a50cd9e76a","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x31b87eaf00d8\u003e: \n resource group \"node-limit-test-2ch85v6j6t94\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2ch85v6j6t94\" to finish deleting: GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n \"name\": \"2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:37:18.781083053Z\",\n \"endTime\": \"2026-10-13T17:01:24.31591322Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0s3j58fj1f3fm4vsgrm1lrp3it3l1 still exists (deletion dispatched at 2026-10-13T16:37:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n correlation ID: 879b6302-da33-4ea2-941e-e29845fe6259\n client request ID: f5a45ff4-173b-4f67-a8d6-7bf91987cdf6\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x31b87eaf00d8\u003e: \n resource group \"node-limit-test-2ch85v6j6t94\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2ch85v6j6t94\" to finish deleting: GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n \"name\": \"2de5bbd0-8d23-4d85-8e58-9564e3649e84\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:37:18.781083053Z\",\n \"endTime\": \"2026-10-13T17:01:24.31591322Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0s3j58fj1f3fm4vsgrm1lrp3it3l1 still exists (deletion dispatched at 2026-10-13T16:37:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9475584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2de5bbd0-8d23-4d85-8e58-9564e3649e84\n correlation ID: 879b6302-da33-4ea2-941e-e29845fe6259\n client request ID: f5a45ff4-173b-4f67-a8d6-7bf91987cdf6\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9f344e165555cc5a4054f01f225300638a3cadc541c865ed8491f7d084dfa29c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110035013809475584","occurred_at":"2026-10-13T15:48:44Z","failed_at":"alert","signature_id":"dc0f6ac01acdec662cd08a68e54dfaa04653844d8ecf2ea1d8d22da7ec3a472c","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T17:22:03Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9475584-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.62:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9475584/providers/microsoft.monitor/accounts/services-j9475584\", namespace=\"klusterlet-2te0sh8st5vlk5p86ivnmeq75dqqc3c7\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gptdj\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2te0sh8st5vlk5p86ivnmeq75dqqc3c7 on cluster ci01-j9475584-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110034877993717760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7503,"pr_state":"closed","pr_sha":"dd38437739ccfb5fe4d5d4bd849164c65a9bad17","final_merged_sha":"dd38437739ccfb5fe4d5d4bd849164c65a9bad17","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T15:48:07Z","started_at":"2026-10-13T15:48:07Z","completed_at":"2026-10-13T17:55:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:06:24.692570763Z","timing_metadata_checked_at":"2026-10-13T18:06:24.692570763Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:06:24.695648253Z","region_metadata_checked_at":"2026-10-13T18:06:24.695648253Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6d01198a01b1e31a95b3f193f7c6d2b0458d660cfb33d9ddab65f824e15430a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110034877993717760","occurred_at":"2026-10-13T15:48:07Z","failed_at":"e2e","signature_id":"0b110e5c27c96f6c961d41c663c021306587e891bbe40929150d9ebda913bc3f","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-bpvl86\" for cluster \"cluster-hs-pre-bpvl86\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d5e48e6be00\u003e: \n failed to create NodePool np-bpvl86: failed waiting for nodepool=\"np-bpvl86\" for cluster \"cluster-hs-pre-bpvl86\" in resourcegroup=\"rg-hs-pre--94dxdngtsz44\" to finish creating: GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\",\n \"name\": \"83a4ff43-825f-4641-9a90-75df22dbb575\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:46:39.5702476Z\",\n \"endTime\": \"2026-10-13T17:15:41.984565887Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--94dxdngtsz44-bpvl86--managed/providers/Microsoft.Compute/virtualMachines/w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--94dxdngtsz44-bpvl86--managed/providers/Microsoft.Compute/virtualMachines/w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\n correlation ID: 4c25f049-32a9-4598-b0f0-3e14c758ee5b\n client request ID: c16c3d79-0122-4655-b7de-468b0afbb13d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-bpvl86\" for cluster \"cluster-hs-pre-bpvl86\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d5e48e6be00\u003e: \n failed to create NodePool np-bpvl86: failed waiting for nodepool=\"np-bpvl86\" for cluster \"cluster-hs-pre-bpvl86\" in resourcegroup=\"rg-hs-pre--94dxdngtsz44\" to finish creating: GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\",\n \"name\": \"83a4ff43-825f-4641-9a90-75df22dbb575\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:46:39.5702476Z\",\n \"endTime\": \"2026-10-13T17:15:41.984565887Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--94dxdngtsz44-bpvl86--managed/providers/Microsoft.Compute/virtualMachines/w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--94dxdngtsz44-bpvl86--managed/providers/Microsoft.Compute/virtualMachines/w2q8t7x4u8t3v0q-np-bpvl86-wnkxp-4lslk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83a4ff43-825f-4641-9a90-75df22dbb575\n correlation ID: 4c25f049-32a9-4598-b0f0-3e14c758ee5b\n client request ID: c16c3d79-0122-4655-b7de-468b0afbb13d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b41227c313c00839417bce35ecb764ba4e0bf0daba507f9fc37b76a4b444ee60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110034877993717760","occurred_at":"2026-10-13T15:48:07Z","failed_at":"e2e","signature_id":"704940cf6bb6426bf6cc6c262b56027ce5f8ecb93098d3e28c4138854cfe377f","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x14c5067047e0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wgpx2l8rb8pt\" to finish creating: GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bfdbd7d1-4678-4b27-805d-06169447d222\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bfdbd7d1-4678-4b27-805d-06169447d222\",\n \"name\": \"bfdbd7d1-4678-4b27-805d-06169447d222\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:50:30.459831291Z\",\n \"endTime\": \"2026-10-13T17:19:31.991486899Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9x2z8c7e8j4t1k-np-delete-main-nf26v-k995z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wgpx2l8rb8pt--managed/providers/Microsoft.Compute/virtualMachines/f9x2z8c7e8j4t1k-np-delete-main-nf26v-k995z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9x2z8c7e8j4t1k-np-delete-main-nf26v-k995z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wgpx2l8rb8pt--managed/providers/Microsoft.Compute/virtualMachines/f9x2z8c7e8j4t1k-np-delete-main-nf26v-k995z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bfdbd7d1-4678-4b27-805d-06169447d222\n correlation ID: 7f043b84-b98d-4c03-bf38-f6fdade9443c\n client request ID: 05384f3e-2de3-496b-af3e-0834a7a0f0d2\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x14c5067047e0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wgpx2l8rb8pt\" to finish creating: GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bfdbd7d1-4678-4b27-805d-06169447d222\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bfdbd7d1-4678-4b27-805d-06169447d222\",\n \"name\": \"bfdbd7d1-4678-4b27-805d-06169447d222\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:50:30.459831291Z\",\n \"endTime\": \"2026-10-13T17:19:31.991486899Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9x2z8c7e8j4t1k-np-delete-main-nf26v-k995z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wgpx2l8rb8pt--managed/providers/Microsoft.Compute/virtualMachines/f9x2z8c7e8j4t1k-np-delete-main-nf26v-k995z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9x2z8c7e8j4t1k-np-delete-main-nf26v-k995z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wgpx2l8rb8pt--managed/providers/Microsoft.Compute/virtualMachines/f9x2z8c7e8j4t1k-np-delete-main-nf26v-k995z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3717760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bfdbd7d1-4678-4b27-805d-06169447d222\n correlation ID: 7f043b84-b98d-4c03-bf38-f6fdade9443c\n client request ID: 05384f3e-2de3-496b-af3e-0834a7a0f0d2","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dpn7kns9h5s6\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110034896461238272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7501,"pr_state":"closed","pr_sha":"7d8c3f4974980ae7204ed927041bb9e824c7452c","final_merged_sha":"7d8c3f4974980ae7204ed927041bb9e824c7452c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T15:47:07Z","started_at":"2026-10-13T15:47:07Z","completed_at":"2026-10-13T18:00:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:16:35.09174031Z","timing_metadata_checked_at":"2026-10-13T18:16:35.09174031Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:16:35.094587701Z","region_metadata_checked_at":"2026-10-13T18:16:35.094587701Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"8d48ecbf6f0a194784e1c4b6d3c58790bb0dda78240d961106c2d5f648dc8943","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110034896461238272","occurred_at":"2026-10-13T15:47:07Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T16:50:39Z\n Ended: 2026-10-13T16:56:39Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1238272-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7497/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110032458631090176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7497,"pr_state":"open","pr_sha":"7bb179778d64a3d2fa0f43ff5a2927cad7511667","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T15:38:35Z","started_at":"2026-10-13T15:38:35Z","completed_at":"2026-10-13T17:58:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:16:34.755363794Z","timing_metadata_checked_at":"2026-10-13T18:16:34.755363794Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:16:34.758221685Z","region_metadata_checked_at":"2026-10-13T18:16:34.758221685Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6092/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110032373646102528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6092,"pr_state":"closed","pr_sha":"6c52f2825f7e760873c725bfc4347e864840e655","final_merged_sha":"6c52f2825f7e760873c725bfc4347e864840e655","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-13T15:38:14Z","started_at":"2026-10-13T15:38:14Z","completed_at":"2026-10-13T18:06:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:16:31.228032143Z","timing_metadata_checked_at":"2026-10-13T18:16:31.228032143Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:16:31.231723832Z","region_metadata_checked_at":"2026-10-13T18:16:31.231723832Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031817837907968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7499,"pr_state":"open","pr_sha":"2302ab8c6efe655af06dcb7f22460973ab22252a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T15:36:01Z","started_at":"2026-10-13T15:36:01Z","completed_at":"2026-10-13T18:02:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:16:34.888252566Z","timing_metadata_checked_at":"2026-10-13T18:16:34.888252566Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:16:34.890764858Z","region_metadata_checked_at":"2026-10-13T18:16:34.890764858Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"2fa1f3590c348f1d8e0c23292a5e6f412efe21b7ea5c32a95f7280f29136c21a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031817837907968","occurred_at":"2026-10-13T15:36:01Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T16:44:34Z\n Ended: 2026-10-13T16:55:36Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7907968-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"ef2ebd7ed137aba03e224893cb03a886ca4776a3aae19c2d218d40e6c1a943ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7499/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031817837907968","occurred_at":"2026-10-13T15:36:01Z","failed_at":"alert","signature_id":"545a94509aea8d174b7c626867e4c3ee62e7a7063a644a5bd20cd3227ea583e7","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T17:32:44Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j7907968-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7907968/providers/microsoft.monitor/accounts/services-j7907968\", namespace=\"klusterlet-2te0u0kbkndd5ajbfrtkgsggtl7jvgde\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-df6h4\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2te0u0kbkndd5ajbfrtkgsggtl7jvgde on cluster ci01-j7907968-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-13T17:36:44Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j7907968-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.161:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7907968/providers/microsoft.monitor/accounts/services-j7907968\", namespace=\"klusterlet-2te0tfu3d0rbgc2bl5u7fshuhidhum5l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-df6h4\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2te0tfu3d0rbgc2bl5u7fshuhidhum5l on cluster ci01-j7907968-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7496,"pr_state":"closed","pr_sha":"619a68d8adf2d7f72abbd588eceb55c08f06be1a","final_merged_sha":"619a68d8adf2d7f72abbd588eceb55c08f06be1a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T15:35:40Z","started_at":"2026-10-13T15:35:40Z","completed_at":"2026-10-13T17:55:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:06:24.287838457Z","timing_metadata_checked_at":"2026-10-13T18:06:24.287838457Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:06:24.290491449Z","region_metadata_checked_at":"2026-10-13T18:06:24.290491449Z"},"failed_at":["alert","e2e"],"failed_test_count":20,"occurrences":[{"row_id":"1570774d96f7b0758179b7d497e915a0b1a9d1cafebc3e12004985347881ff9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"0bee45c13efb267807d223dc42fdb6b42774c050c8b11335bf9553e927e7a784","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-x7n4rm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1df325fdde20\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq\" in resourcegroup=\"rg-candidate-4-23-25wqkbxjvqpm\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n correlation ID: 66a77b00-0c74-46a1-bab1-d73fd9784240\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:112]: failed to create customer resources for cluster \"cluster-candidate-4-23-x7n4rm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1df325fdde20\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq\" in resourcegroup=\"rg-candidate-4-23-25wqkbxjvqpm\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-23-25wqkbxjvqpm/providers/Microsoft.Resources/deployments/customer-infra-cluster-candidate-4-23-x7n4rm-mgfdjq/operationStatuses/08584096986588873020\n correlation ID: 66a77b00-0c74-46a1-bab1-d73fd9784240\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"161166c21b0d647ccb3d60903e346d64f8235244e6c8f6a833ff32ad2ad583ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"alert","signature_id":"d7646a570dc56c8aaf2ff17561f2018e209352457d4d221d6f98298c58002476","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T17:33:37Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j6938112-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.198:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6938112/providers/microsoft.monitor/accounts/services-j6938112\", namespace=\"klusterlet-2te0oroei8rqcmb2im244arlb9dt9evs\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-ztvsb\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2te0oroei8rqcmb2im244arlb9dt9evs on cluster ci01-j6938112-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"1e19f17ccced0706dd4da010eafb082dbd62bcdcc90d0629a3ae83073f930191","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"6d01e4638c3973f17302f07ed79508b57643e22d74bb892cac9c1af52e92a292","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x309531b0198\u003e: \n resource group \"ea-list-wtljvtmmhcsk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list8h4wvd\" in resourcegroup=\"ea-list-wtljvtmmhcsk\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\",\n \"name\": \"2f74cf89-c5af-4bbc-9410-e14470aca714\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:38:18.149454561Z\",\n \"endTime\": \"2026-10-13T17:02:21.62230875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o6snbp7v915nacod7cib95rmn96j still exists (deletion dispatched at 2026-10-13T16:38:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n correlation ID: 8d339200-f0e0-47b8-90d7-53a61a990c91\n client request ID: 742fa150-0903-4156-b9ea-5d97b505f86c\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x309531b0198\u003e: \n resource group \"ea-list-wtljvtmmhcsk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-list8h4wvd\" in resourcegroup=\"ea-list-wtljvtmmhcsk\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\",\n \"name\": \"2f74cf89-c5af-4bbc-9410-e14470aca714\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:38:18.149454561Z\",\n \"endTime\": \"2026-10-13T17:02:21.62230875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o6snbp7v915nacod7cib95rmn96j still exists (deletion dispatched at 2026-10-13T16:38:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2f74cf89-c5af-4bbc-9410-e14470aca714\n correlation ID: 8d339200-f0e0-47b8-90d7-53a61a990c91\n client request ID: 742fa150-0903-4156-b9ea-5d97b505f86c\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"20a9c0db5927d7994d7faaf36ea320390d30c293aa65b83281bd7e541277a0c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"d91ceb0c461dd8cc9e4f63cf5a15d575daca06c95808c7ef0af47faf649a13d6","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x86c6d66c288\u003e: \n resource group \"ephemeral-osdisk-zk6g6jlt9pcm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-zk6g6jlt9pcm\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e42b64f2-dbc7-46c0-ade3-38701934a844\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e42b64f2-dbc7-46c0-ade3-38701934a844\",\n \"name\": \"e42b64f2-dbc7-46c0-ade3-38701934a844\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:37:24.341394968Z\",\n \"endTime\": \"2026-10-13T17:01:31.660742691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o0fo95vc6ql30rb17lllmie9v71c still exists (deletion dispatched at 2026-10-13T16:37:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e42b64f2-dbc7-46c0-ade3-38701934a844\n correlation ID: c01082db-7899-4024-8f97-9610dd939a4e\n client request ID: 1f3a6602-152b-48ad-934d-22af532614cf\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x86c6d66c288\u003e: \n resource group \"ephemeral-osdisk-zk6g6jlt9pcm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-zk6g6jlt9pcm\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e42b64f2-dbc7-46c0-ade3-38701934a844\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e42b64f2-dbc7-46c0-ade3-38701934a844\",\n \"name\": \"e42b64f2-dbc7-46c0-ade3-38701934a844\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:37:24.341394968Z\",\n \"endTime\": \"2026-10-13T17:01:31.660742691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o0fo95vc6ql30rb17lllmie9v71c still exists (deletion dispatched at 2026-10-13T16:37:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e42b64f2-dbc7-46c0-ade3-38701934a844\n correlation ID: c01082db-7899-4024-8f97-9610dd939a4e\n client request ID: 1f3a6602-152b-48ad-934d-22af532614cf\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"392f0e827d93396e098958fa1da86b2bee741ad3b425fcd95e260766c43c2f4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"ecb5ffa542f87571700262dae2b7c84f4536526d903a70c009dcccac42adf516","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x36cea0d80120\u003e: \n resource group \"rg-cluster-back-version-4285twt9lhf2\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x36ce9f324380\u003e{\n msg: \"resource group \\\"rg-cluster-back-version-4285twt9lhf2\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x36ce9f324360\u003e{\n msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n err: \u003c*fmt.wrapErrors | 0x36cea0a86750\u003e{\n msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x36cea0d80120\u003e: \n resource group \"rg-cluster-back-version-4285twt9lhf2\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x36ce9f324380\u003e{\n msg: \"resource group \\\"rg-cluster-back-version-4285twt9lhf2\\\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x36ce9f324360\u003e{\n msg: \"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n err: \u003c*fmt.wrapErrors | 0x36cea0a86750\u003e{\n msg: \"failed deleting hcp clusters in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\", caused by: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n \u003c*fmt.wrapErrors | 0x36ce9f315470\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-4285twt9lhf2\\\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0x36ce9f860ef0\u003e{\n s: \"timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"resource group \"rg-cluster-back-version-4285twt9lhf2\": failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-4285twt9lhf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-4285twt9lhf2, error: context deadline exceeded"}},{"row_id":"60929b2a4951ae5e29d4be839f4c2a8c35f7df74157b88a278cc51264da85f1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"72c1ad55e4062aec518f3f62817b93e6395a0432a25afcb61cb220d43060d265","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x2e91a04e0060\u003e: \n resource group \"patch-tags-4btcwtspkn4z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4btcwtspkn4z\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/487410f3-b72a-4fc3-9193-c1517e85efb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/487410f3-b72a-4fc3-9193-c1517e85efb3\",\n \"name\": \"487410f3-b72a-4fc3-9193-c1517e85efb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:37:24.68179111Z\",\n \"endTime\": \"2026-10-13T17:01:31.676853999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nltmn5utnsjpt0ljsht3622t7msm still exists (deletion dispatched at 2026-10-13T16:37:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/487410f3-b72a-4fc3-9193-c1517e85efb3\n correlation ID: 39a9fc6d-ba22-4803-9010-4b78f2a6492a\n client request ID: 8d74e002-092e-41b2-a1b9-73b32b63b7fa\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x2e91a04e0060\u003e: \n resource group \"patch-tags-4btcwtspkn4z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4btcwtspkn4z\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/487410f3-b72a-4fc3-9193-c1517e85efb3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/487410f3-b72a-4fc3-9193-c1517e85efb3\",\n \"name\": \"487410f3-b72a-4fc3-9193-c1517e85efb3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:37:24.68179111Z\",\n \"endTime\": \"2026-10-13T17:01:31.676853999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nltmn5utnsjpt0ljsht3622t7msm still exists (deletion dispatched at 2026-10-13T16:37:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/487410f3-b72a-4fc3-9193-c1517e85efb3\n correlation ID: 39a9fc6d-ba22-4803-9010-4b78f2a6492a\n client request ID: 8d74e002-092e-41b2-a1b9-73b32b63b7fa\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6dc19e0c6b6321f856f66d8ba40687fd9fa27e036fae18cf475244813f4e3e36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"ecc5ff218b668fca411a6a387f8fa1e8cc8fcb2c2b7b038bd92d4f9b96814b90","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3646ea5bc090\u003e: \n resource group \"node-limit-test-8kxdfg5cbrwp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8kxdfg5cbrwp\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5a7b5709-9d05-492c-bf2b-e88948da705b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5a7b5709-9d05-492c-bf2b-e88948da705b\",\n \"name\": \"5a7b5709-9d05-492c-bf2b-e88948da705b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:38:27.169390438Z\",\n \"endTime\": \"2026-10-13T17:02:31.60705305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nvvfqaanfc2udupbrnjbsmglpsr1 still exists (deletion dispatched at 2026-10-13T16:38:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5a7b5709-9d05-492c-bf2b-e88948da705b\n correlation ID: 8e472778-6c92-4350-b17d-60c0f029163b\n client request ID: abc90e83-96e9-4095-a014-84d0c0f96649\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3646ea5bc090\u003e: \n resource group \"node-limit-test-8kxdfg5cbrwp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8kxdfg5cbrwp\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5a7b5709-9d05-492c-bf2b-e88948da705b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5a7b5709-9d05-492c-bf2b-e88948da705b\",\n \"name\": \"5a7b5709-9d05-492c-bf2b-e88948da705b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:38:27.169390438Z\",\n \"endTime\": \"2026-10-13T17:02:31.60705305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nvvfqaanfc2udupbrnjbsmglpsr1 still exists (deletion dispatched at 2026-10-13T16:38:28Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5a7b5709-9d05-492c-bf2b-e88948da705b\n correlation ID: 8e472778-6c92-4350-b17d-60c0f029163b\n client request ID: abc90e83-96e9-4095-a014-84d0c0f96649\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"73737ca27182523795624709ff3654e2f0a3d2594d3d52908a197ea01213e30a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"94aae066f49ff828f8658c561f57e2a599bd9f4bcee2e2d506ad8011ab4f1278","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3c5bff2321b0\u003e: \n resource group \"rg-candidate-4-21-ckfqmkfrhb7s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-x2ltjj\" in resourcegroup=\"rg-candidate-4-21-ckfqmkfrhb7s\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/35900dd3-f76a-4ad6-8666-ea0b4b1885cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/35900dd3-f76a-4ad6-8666-ea0b4b1885cd\",\n \"name\": \"35900dd3-f76a-4ad6-8666-ea0b4b1885cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:36:24.431313337Z\",\n \"endTime\": \"2026-10-13T17:00:31.622971609Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o6eoe74gqn4aq2k2plogghjg4a3t still exists (deletion dispatched at 2026-10-13T16:36:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/35900dd3-f76a-4ad6-8666-ea0b4b1885cd\n correlation ID: 8a956502-ed8a-4638-9158-d916f64e4232\n client request ID: 5f464c51-9b3d-4872-ac2e-f49527555192\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3c5bff2321b0\u003e: \n resource group \"rg-candidate-4-21-ckfqmkfrhb7s\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-x2ltjj\" in resourcegroup=\"rg-candidate-4-21-ckfqmkfrhb7s\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/35900dd3-f76a-4ad6-8666-ea0b4b1885cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/35900dd3-f76a-4ad6-8666-ea0b4b1885cd\",\n \"name\": \"35900dd3-f76a-4ad6-8666-ea0b4b1885cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:36:24.431313337Z\",\n \"endTime\": \"2026-10-13T17:00:31.622971609Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o6eoe74gqn4aq2k2plogghjg4a3t still exists (deletion dispatched at 2026-10-13T16:36:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/35900dd3-f76a-4ad6-8666-ea0b4b1885cd\n correlation ID: 8a956502-ed8a-4638-9158-d916f64e4232\n client request ID: 5f464c51-9b3d-4872-ac2e-f49527555192\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7c7ca3e4a86a53af5a07331f361cbc67995cd1ac21916c324945d08d1380cf43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"669ef81600d88868158f50a61a32eb55792068d452440bf21253695791921ce2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x223786a70108\u003e: \n resource group \"rg-candidate-5-0-l4hscszf2rxz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-lvqr8d\" in resourcegroup=\"rg-candidate-5-0-l4hscszf2rxz\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\",\n \"name\": \"b44ea302-7f97-4969-a60c-eec855852cac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:40:42.747363753Z\",\n \"endTime\": \"2026-10-13T17:04:51.623105937Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o054eslakn49l68pthqigeds4ule still exists (deletion dispatched at 2026-10-13T16:40:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n correlation ID: 4bfc0959-e629-40fb-9245-01ec4651bd6b\n client request ID: a781f5bf-303e-4280-ba8a-44cc833b6ce4\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x223786a70108\u003e: \n resource group \"rg-candidate-5-0-l4hscszf2rxz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-lvqr8d\" in resourcegroup=\"rg-candidate-5-0-l4hscszf2rxz\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\",\n \"name\": \"b44ea302-7f97-4969-a60c-eec855852cac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:40:42.747363753Z\",\n \"endTime\": \"2026-10-13T17:04:51.623105937Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o054eslakn49l68pthqigeds4ule still exists (deletion dispatched at 2026-10-13T16:40:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b44ea302-7f97-4969-a60c-eec855852cac\n correlation ID: 4bfc0959-e629-40fb-9245-01ec4651bd6b\n client request ID: a781f5bf-303e-4280-ba8a-44cc833b6ce4\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8ef948b52bb8b06d7775adaf234155fcad8b3fba4d2416825b0bcb374f6e907f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"7be575f2f85e63d8ce2308f15291aa10c120d9bccad74cdd55d95042f97a55a5","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x33c5e58a20c0\u003e: \n resource group \"private-keyvault-tbl82js9plw4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-tbl82js9plw4\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e616d6b4-805f-435f-aa67-d2ddddd5c316\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e616d6b4-805f-435f-aa67-d2ddddd5c316\",\n \"name\": \"e616d6b4-805f-435f-aa67-d2ddddd5c316\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:35:08.583263738Z\",\n \"endTime\": \"2026-10-13T16:59:11.97611737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0op2sf7jll6tp2tplqbte694tvbr6 still exists (deletion dispatched at 2026-10-13T16:35:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e616d6b4-805f-435f-aa67-d2ddddd5c316\n correlation ID: 41573f31-0b63-4f6a-92a1-29c5b34c8e25\n client request ID: 94ebdfbe-0afd-4bb5-b0fc-3201e548fa3c\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x33c5e58a20c0\u003e: \n resource group \"private-keyvault-tbl82js9plw4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-tbl82js9plw4\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e616d6b4-805f-435f-aa67-d2ddddd5c316\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e616d6b4-805f-435f-aa67-d2ddddd5c316\",\n \"name\": \"e616d6b4-805f-435f-aa67-d2ddddd5c316\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:35:08.583263738Z\",\n \"endTime\": \"2026-10-13T16:59:11.97611737Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0op2sf7jll6tp2tplqbte694tvbr6 still exists (deletion dispatched at 2026-10-13T16:35:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e616d6b4-805f-435f-aa67-d2ddddd5c316\n correlation ID: 41573f31-0b63-4f6a-92a1-29c5b34c8e25\n client request ID: 94ebdfbe-0afd-4bb5-b0fc-3201e548fa3c\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ad1fa214a400925a867f8579ac998b1d691331acdb9f8002e8bf07347fb9f9c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"8d5ca072ced2a86e224e1159c22de688aa029c8ce11fd664e12553f4ed9b8ecd","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x343c841917c0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-ld8wft\" in resourcegroup=\"rg-np-noedge-ld8wft-lrx6cgcqbn7h\" to finish creating: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n correlation ID: ffecaa7d-bd14-4598-8eba-697dbec64dbe\n client request ID: 41765bff-079d-45be-ae54-98a967650838\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x343c841917c0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-ld8wft\" in resourcegroup=\"rg-np-noedge-ld8wft-lrx6cgcqbn7h\" to finish creating: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6b0653ae-307f-47d1-8798-9271b8c88bd6\n correlation ID: ffecaa7d-bd14-4598-8eba-697dbec64dbe\n client request ID: 41765bff-079d-45be-ae54-98a967650838\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}},{"row_id":"b4ab07b22b578c0bac9b95ae804e2324a33083964becb1089c0532ab6663601c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"bf1c9ad85dc67c52bb5db81e418c326fc84cc55cd5304ca050cd7f99526fd13a","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x1bf360d7c078\u003e: \n resource group \"delayed-rbac-86jnlmj4hp5c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-86jnlmj4hp5c\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b559cd85-38ce-4c5c-97e2-77f3a2b7b2b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b559cd85-38ce-4c5c-97e2-77f3a2b7b2b4\",\n \"name\": \"b559cd85-38ce-4c5c-97e2-77f3a2b7b2b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:32:32.577215674Z\",\n \"endTime\": \"2026-10-13T16:56:41.960331085Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nogn7v488ab5md5u7qq776j5f8gl still exists (deletion dispatched at 2026-10-13T16:32:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b559cd85-38ce-4c5c-97e2-77f3a2b7b2b4\n correlation ID: 260d0473-ba98-4907-80ca-4905c6dbb05b\n client request ID: dca2c435-4751-4ccd-b2b2-4c228a5df5fa\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x1bf360d7c078\u003e: \n resource group \"delayed-rbac-86jnlmj4hp5c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-86jnlmj4hp5c\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b559cd85-38ce-4c5c-97e2-77f3a2b7b2b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b559cd85-38ce-4c5c-97e2-77f3a2b7b2b4\",\n \"name\": \"b559cd85-38ce-4c5c-97e2-77f3a2b7b2b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:32:32.577215674Z\",\n \"endTime\": \"2026-10-13T16:56:41.960331085Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nogn7v488ab5md5u7qq776j5f8gl still exists (deletion dispatched at 2026-10-13T16:32:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b559cd85-38ce-4c5c-97e2-77f3a2b7b2b4\n correlation ID: 260d0473-ba98-4907-80ca-4905c6dbb05b\n client request ID: dca2c435-4751-4ccd-b2b2-4c228a5df5fa\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b9693e649cd6ed536055af827be7460995ab942863c7cd57270c0ac91052dbb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"c9a49014768753cae5a2b15e7de520cc636773dabb5a420f9a78aef34c0146ac","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-gpscnh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x19351c9c25a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n correlation ID: 73cd0e3a-2347-4fb2-a937-dc27e30c4de2\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:123]: failed to create customer resources for y-stream upgrade cluster \"cp-ystream-upgrade-4-21-gpscnh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x19351c9c25a0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cp-ystream-upgrade-4-21-q2fxp7w8nsb4/providers/Microsoft.Resources/deployments/customer-infra-cp-ystream-upgrade-4-21-gpscnh-6d9mtn/operationStatuses/08584096986507070624\n correlation ID: 73cd0e3a-2347-4fb2-a937-dc27e30c4de2\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"c8bcf0bd263c68f8a4d4603a4c3df6f622c5e3829e3f4567d9f370ed3c62638f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"e5551334aa432c7f1d4fcaa4df0fbb0de936bcd598badb4046f07fb57beb7449","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0xe966a774240\u003e: \n resource group \"arm64-vm-cluster-6rwgx9dzrn7g\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-6rwgx9dzrn7g\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2b094443-c590-42b9-980a-362abc782d9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2b094443-c590-42b9-980a-362abc782d9e\",\n \"name\": \"2b094443-c590-42b9-980a-362abc782d9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:34:49.54918973Z\",\n \"endTime\": \"2026-10-13T16:58:51.683203613Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nl800tudbvnpa73iohd0jebru9cn still exists (deletion dispatched at 2026-10-13T16:34:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2b094443-c590-42b9-980a-362abc782d9e\n correlation ID: f88d95b7-3797-479d-9891-e9a44575f535\n client request ID: c34e4445-2545-411b-9c29-5bbb95e1b4bb\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0xe966a774240\u003e: \n resource group \"arm64-vm-cluster-6rwgx9dzrn7g\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-6rwgx9dzrn7g\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2b094443-c590-42b9-980a-362abc782d9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2b094443-c590-42b9-980a-362abc782d9e\",\n \"name\": \"2b094443-c590-42b9-980a-362abc782d9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:34:49.54918973Z\",\n \"endTime\": \"2026-10-13T16:58:51.683203613Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nl800tudbvnpa73iohd0jebru9cn still exists (deletion dispatched at 2026-10-13T16:34:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2b094443-c590-42b9-980a-362abc782d9e\n correlation ID: f88d95b7-3797-479d-9891-e9a44575f535\n client request ID: c34e4445-2545-411b-9c29-5bbb95e1b4bb\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e71bf281a2734d4a20d9a215c73da7b1f70d485cd01628f2ab1962524f4a031a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"a73e237dc986d50343fffd30aca449541788309f42293d44e9183312dbe9d419","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x2c1d48a96060\u003e: \n resource group \"disabled-image-registry-7ddljxcvwjcm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7ddljxcvwjcm\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n \"name\": \"bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:38:07.060491044Z\",\n \"endTime\": \"2026-10-13T17:02:11.454465514Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n correlation ID: bc63c490-e9f1-483e-a3e0-dbf6c6ddf72f\n client request ID: 93e5d8b4-b42f-4292-ab79-f0ec030e5a84\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x2c1d48a96060\u003e: \n resource group \"disabled-image-registry-7ddljxcvwjcm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7ddljxcvwjcm\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n \"name\": \"bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:38:07.060491044Z\",\n \"endTime\": \"2026-10-13T17:02:11.454465514Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bf67735b-40e3-4e9f-b12b-b1b7b58e9b9e\n correlation ID: bc63c490-e9f1-483e-a3e0-dbf6c6ddf72f\n client request ID: 93e5d8b4-b42f-4292-ab79-f0ec030e5a84\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"edca5cdcd4a72305c40a2e4a538054f8afb96d62c64ddd51014983f00f21f1f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"022aadd1c5b43f5fcb46b49de5bea32873432564b44b704b42ed859546fe8e82","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3bf37649c048\u003e: \n resource group \"customer-rg-kptnhtffptgl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-kptnhtffptgl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n \"name\": \"d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:42:39.455530141Z\",\n \"endTime\": \"2026-10-13T17:06:41.723960509Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0oaptqtlfm45pi2ru69verofh93nj still exists (deletion dispatched at 2026-10-13T16:42:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n correlation ID: 57954ad9-2ba3-4943-8c75-f5745f7ae1f7\n client request ID: bbb4049f-9eb5-4ee1-9bdd-ccda4efc41c1\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x3bf37649c048\u003e: \n resource group \"customer-rg-kptnhtffptgl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-kptnhtffptgl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n \"name\": \"d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:42:39.455530141Z\",\n \"endTime\": \"2026-10-13T17:06:41.723960509Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0oaptqtlfm45pi2ru69verofh93nj still exists (deletion dispatched at 2026-10-13T16:42:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d3fd105c-2a6a-4db9-8a7d-c2c20e4d8b1e\n correlation ID: 57954ad9-2ba3-4943-8c75-f5745f7ae1f7\n client request ID: bbb4049f-9eb5-4ee1-9bdd-ccda4efc41c1\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f150b6072b0a3897eac739e93540451055cf2dd8d88a15986d738a6c14229cb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"827a5564727e582e9b07423822965a3f6d9902a3cc857595f1ab3814f76516c1","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x34e38763a138\u003e: \n resource group \"rg-cordon-test-9l4cgfh8mhfl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-9l4cgfh8mhfl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n \"name\": \"84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:39:24.254400328Z\",\n \"endTime\": \"2026-10-13T17:03:31.595626382Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o1agno7465r2vsakhkack392k7ql still exists (deletion dispatched at 2026-10-13T16:39:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n correlation ID: 9b32bec0-6d45-4533-a8ad-d586fd5de17c\n client request ID: 0c2d0a32-aa6c-4c82-a1b0-303746bfb85e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x34e38763a138\u003e: \n resource group \"rg-cordon-test-9l4cgfh8mhfl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-9l4cgfh8mhfl\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n \"name\": \"84ab95bc-886b-4b3d-9a6e-9221165e46bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:39:24.254400328Z\",\n \"endTime\": \"2026-10-13T17:03:31.595626382Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0o1agno7465r2vsakhkack392k7ql still exists (deletion dispatched at 2026-10-13T16:39:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/84ab95bc-886b-4b3d-9a6e-9221165e46bd\n correlation ID: 9b32bec0-6d45-4533-a8ad-d586fd5de17c\n client request ID: 0c2d0a32-aa6c-4c82-a1b0-303746bfb85e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f54e92a92df51c6b47f5e293f3e718d9ee2915424c159838447ff9e53649d72c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"b92ba843e0e832a2ad56a0bbd96074f26f53e9816edef7f8e3b6ac20d8300aff","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x2f1a7aab6150\u003e: \n resource group \"rg-candidate-4-22-lskfpcc9pllz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-d2g9r5\" in resourcegroup=\"rg-candidate-4-22-lskfpcc9pllz\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f9ed645a-3dcf-42dc-9141-f0f20d1df080\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f9ed645a-3dcf-42dc-9141-f0f20d1df080\",\n \"name\": \"f9ed645a-3dcf-42dc-9141-f0f20d1df080\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:36:46.601277196Z\",\n \"endTime\": \"2026-10-13T17:00:51.586929721Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nlj053nsa11ldfoisafgipanrlce still exists (deletion dispatched at 2026-10-13T16:36:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f9ed645a-3dcf-42dc-9141-f0f20d1df080\n correlation ID: 39b6fd78-ce95-48e2-8999-56a159d15d97\n client request ID: fa481482-679a-4e18-8a64-9dbb028a0bbd\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x2f1a7aab6150\u003e: \n resource group \"rg-candidate-4-22-lskfpcc9pllz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-d2g9r5\" in resourcegroup=\"rg-candidate-4-22-lskfpcc9pllz\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f9ed645a-3dcf-42dc-9141-f0f20d1df080\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f9ed645a-3dcf-42dc-9141-f0f20d1df080\",\n \"name\": \"f9ed645a-3dcf-42dc-9141-f0f20d1df080\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:36:46.601277196Z\",\n \"endTime\": \"2026-10-13T17:00:51.586929721Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nlj053nsa11ldfoisafgipanrlce still exists (deletion dispatched at 2026-10-13T16:36:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f9ed645a-3dcf-42dc-9141-f0f20d1df080\n correlation ID: 39b6fd78-ce95-48e2-8999-56a159d15d97\n client request ID: fa481482-679a-4e18-8a64-9dbb028a0bbd\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f5f0431eb961fce19b738abd1a053f6ea269e0c8b548b2faa6288c74aed2b001","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"a17143f3d84444b29b8010bee708abc0adf653cb05b16055bea351d2e9fb5d99","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x1722b271c198\u003e: \n resource group \"v20260901preview-5x7tmrznp2t6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-5x7tmrznp2t6\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4cd1ca19-0014-4b36-b986-419e0d5b84e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4cd1ca19-0014-4b36-b986-419e0d5b84e1\",\n \"name\": \"4cd1ca19-0014-4b36-b986-419e0d5b84e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:36:15.001844938Z\",\n \"endTime\": \"2026-10-13T17:00:21.681973204Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nlh1b0gebk9buu1868a057tlqheh still exists (deletion dispatched at 2026-10-13T16:36:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4cd1ca19-0014-4b36-b986-419e0d5b84e1\n correlation ID: 78699c18-7077-4978-876a-af57a1f9b864\n client request ID: 3c23b588-05ad-4ee2-9c29-3eca24b5d20c\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x1722b271c198\u003e: \n resource group \"v20260901preview-5x7tmrznp2t6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-5x7tmrznp2t6\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4cd1ca19-0014-4b36-b986-419e0d5b84e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4cd1ca19-0014-4b36-b986-419e0d5b84e1\",\n \"name\": \"4cd1ca19-0014-4b36-b986-419e0d5b84e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:36:15.001844938Z\",\n \"endTime\": \"2026-10-13T17:00:21.681973204Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nlh1b0gebk9buu1868a057tlqheh still exists (deletion dispatched at 2026-10-13T16:36:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4cd1ca19-0014-4b36-b986-419e0d5b84e1\n correlation ID: 78699c18-7077-4978-876a-af57a1f9b864\n client request ID: 3c23b588-05ad-4ee2-9c29-3eca24b5d20c\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ff1dbec92f8e241ca7e708a4538baf9d9cb58cfcf79cc2efe6fc69e74fd9abf8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7496/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110031760526938112","occurred_at":"2026-10-13T15:35:40Z","failed_at":"e2e","signature_id":"cd7a71f4ca6cd6df4fa5f6e8e1c80b538d136ae3f5c114c7ba728953809926e5","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x39bb325a6558\u003e: \n resource group \"admin-api-serialconsole-dwh828chkx9c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-dwh828chkx9c\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n \"name\": \"0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:33:20.010489415Z\",\n \"endTime\": \"2026-10-13T16:57:21.684870881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nldrfocqarmt0r34ht6jph1m9d19 still exists (deletion dispatched at 2026-10-13T16:33:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n correlation ID: d6c08569-5a60-4a0f-bb76-207075ebe9e3\n client request ID: f4713fe8-8c37-423d-a5df-6a24a2af5b2d\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:309]: Unexpected error:\n \u003c*errors.joinError | 0x39bb325a6558\u003e: \n resource group \"admin-api-serialconsole-dwh828chkx9c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-dwh828chkx9c\" to finish deleting: GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n \"name\": \"0074eb0c-2b24-41cb-a7ff-604c4e76ad09\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:33:20.010489415Z\",\n \"endTime\": \"2026-10-13T16:57:21.684870881Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2te0nldrfocqarmt0r34ht6jph1m9d19 still exists (deletion dispatched at 2026-10-13T16:33:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6938112.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0074eb0c-2b24-41cb-a7ff-604c4e76ad09\n correlation ID: d6c08569-5a60-4a0f-bb76-207075ebe9e3\n client request ID: f4713fe8-8c37-423d-a5df-6a24a2af5b2d\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":20,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504"],"matched_occurrences":20,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7498/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110067024905375744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7498,"pr_state":"open","pr_sha":"eea5515edceceeab7a799999f9169ec68a59a3c8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T15:32:40Z","started_at":"2026-10-13T15:32:40Z","completed_at":"2026-10-13T20:02:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T20:12:33.583200368Z","timing_metadata_checked_at":"2026-10-13T20:12:33.583200368Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T20:12:33.585627561Z","region_metadata_checked_at":"2026-10-13T20:12:33.585627561Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110030920055525376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7501,"pr_state":"closed","pr_sha":"7d8c3f4974980ae7204ed927041bb9e824c7452c","final_merged_sha":"7d8c3f4974980ae7204ed927041bb9e824c7452c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T15:32:24Z","started_at":"2026-10-13T15:32:24Z","completed_at":"2026-10-13T15:48:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T16:21:20.620963554Z","timing_metadata_checked_at":"2026-10-13T16:21:20.620963554Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T16:21:20.626191038Z","region_metadata_checked_at":"2026-10-13T16:21:20.626191038Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"8833293d7fd54ed42efaaf4832f414900a8bf661ff32c8351c28f703b3fe6827","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110030920055525376","occurred_at":"2026-10-13T15:32:24Z","failed_at":"alert","signature_id":"a13b64d94d488082a4a9ffd8fdeb35b5d5d0a71c93381c3fedbe0bba79b488d1","test_name":"[aro-hcp-observability] [infra] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"b92982ce36e6e8480364ad24a90b56398ce909a3b287cc52f5f660b1227c9277","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110030920055525376","occurred_at":"2026-10-13T15:32:24Z","failed_at":"alert","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"c87112c4edca1a6d851721370b53083457a0f7f64cae44600f00aa0753a933ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7501/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110030920055525376","occurred_at":"2026-10-13T15:32:24Z","failed_at":"alert","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","test_name":"[aro-hcp-observability] [svc] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110029189787684864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7495,"pr_state":"open","pr_sha":"3c936e16bdf98e15f0724566b1efeaef8d31a693","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T15:25:35Z","started_at":"2026-10-13T15:25:35Z","completed_at":"2026-10-13T17:38:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T17:56:19.810845146Z","timing_metadata_checked_at":"2026-10-13T17:56:19.810845146Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T17:56:19.813656036Z","region_metadata_checked_at":"2026-10-13T17:56:19.813656036Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"529dfc4b74c51c3538d1ce1aca734827a09dfd7097e0e7771058153bd6f40973","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110029189787684864","occurred_at":"2026-10-13T15:25:35Z","failed_at":"alert","signature_id":"67c3460376d9d8bd0e4afb70adcef578e97c7095d3636ddd63015dceac05c2d7","test_name":"[aro-hcp-observability] [infra] alert Cosmos DB Normalized RU Consumption High - arohcpci01-rp-j7684864 does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T16:36:18Z\n Ended: 2026-10-13T16:55:14Z\n Severity: Sev3\n Description: Cosmos DB normalized RU consumption is above 70% averaged over a 15-minute window, evaluated every minute. Investigate workload patterns or increase provisioned throughput. https://learn.microsoft.com/azure/cosmos-db/monitor-normalized-request-units","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","failure_pattern":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired","search_query":"alert [infra] Cosmos DB Normalized RU Consumption High - \u003ccosmos-account\u003e fired"}},{"row_id":"e06f5377a01ffd00f3d599bf2894caff2582efa33ec084decebc7e050646db5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7495/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110029189787684864","occurred_at":"2026-10-13T15:25:35Z","failed_at":"e2e","signature_id":"0af58b2f76d489b90f234310d2a3ac8b142182a0239f813bb77188e32f9dc462","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-cvz2rwgtfkl8/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-cvz2rwgtfkl8/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-c5gl2m96t572/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4275a314d89bf06a724f7bf3c5bfe06406a90fa00482d282b46f40a1b17ff273","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110028695728033792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7469,"pr_state":"closed","pr_sha":"a2d7fd940b1ba037c80cee86cf5b380e84ab4193","final_merged_sha":"a2d7fd940b1ba037c80cee86cf5b380e84ab4193","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T15:23:37Z","started_at":"2026-10-13T15:23:37Z","completed_at":"2026-10-13T17:56:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:06:23.516825224Z","timing_metadata_checked_at":"2026-10-13T18:06:23.516825224Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:06:23.519582515Z","region_metadata_checked_at":"2026-10-13T18:06:23.519582515Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"14364924a9dc8b902bd1c7646564ad8e5f9bee3aaa9a366256db2182e7069b0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110028695728033792","occurred_at":"2026-10-13T15:23:37Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T16:31:51Z\n Ended: 2026-10-13T16:42:49Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8033792-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110028457319600128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7451,"pr_state":"open","pr_sha":"4356f88c518a199488f8f45f3f21d185b3435491","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T15:22:41Z","started_at":"2026-10-13T15:22:41Z","completed_at":"2026-10-13T17:56:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:06:23.10231955Z","timing_metadata_checked_at":"2026-10-13T18:06:23.10231955Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:06:23.105952839Z","region_metadata_checked_at":"2026-10-13T18:06:23.105952839Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"9536eee083c759e41279b11e5a1e3374e8a2f30913e318bf17ac9e3d63be8a82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7451/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110028457319600128","occurred_at":"2026-10-13T15:22:41Z","failed_at":"alert","signature_id":"873a799defb32957e4db008041c588dd1cd17af0950d5ca06af312d9239ed0f0","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T16:29:41Z\n Ended: 2026-10-13T16:48:44Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9600128-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7485,"pr_state":"closed","pr_sha":"c6f94bff94d53889c16cfbe68c2fbf2f7e27f72c","final_merged_sha":"2c6f6382730d69d9ad5231a907568292ded88b2c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T15:15:38Z","started_at":"2026-10-13T15:15:38Z","completed_at":"2026-10-13T17:55:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T18:06:23.943666659Z","timing_metadata_checked_at":"2026-10-13T18:06:23.943666659Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T18:06:23.94641295Z","region_metadata_checked_at":"2026-10-13T18:06:23.94641295Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"368a57cc36cfa0262cf4c815cadfeb7df6f757f2315c1473d68f964afe988db4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","failed_at":"alert","signature_id":"683b9a267c4b1dc9692126fc62980df123b055e51ab1e62c31bcb909366380a0","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T17:24:10Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9411968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9411968/providers/microsoft.monitor/accounts/hcps-j9411968\", namespace=\"ocm-arohcpci01-2te0nj9h3askcfok1isadggsor76sejq-x1r3f9l2r1b0r9d\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-dj7ht\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te0nj9h3askcfok1isadggsor76sejq-x1r3f9l2r1b0r9d/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-13T17:24:10Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9411968-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9411968/providers/microsoft.monitor/accounts/hcps-j9411968\", namespace=\"ocm-arohcpci01-2te0nj9h3askcfok1isadggsor76sejq-x1r3f9l2r1b0r9d\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-dj7ht\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2te0nj9h3askcfok1isadggsor76sejq-x1r3f9l2r1b0r9d/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"8cd6ecd6cfd4ddf99180c8a2f885e299f7880ad6c2acbd656dd5e5a897ba1bea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","failed_at":"alert","signature_id":"be3f2336b804bb116f4220473f3b514fdb2978302e5dac315a122f4afc722704","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T16:51:06Z\n Ended: 2026-10-13T17:10:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9411968-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.192:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9411968/providers/microsoft.monitor/accounts/hcps-j9411968\", namespace=\"ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0749a532-89d2-4bdf-99d2-f75e8cbffe37\"\n Description: Pod ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-13T16:51:06Z\n Ended: 2026-10-13T17:10:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9411968-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.192:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9411968/providers/microsoft.monitor/accounts/hcps-j9411968\", namespace=\"ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"0749a532-89d2-4bdf-99d2-f75e8cbffe37\"\n Description: Pod ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"bae4903d07abfae022929d39bd5df4b5d87c21caeb4928162fdbad3ff33fbdaf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","failed_at":"alert","signature_id":"812fdff806dd337434b39662e19b80b3cf432a7b1671571c261b606d2d5fc679","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T16:48:12Z\n Ended: 2026-10-13T17:05:12Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9411968-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.192:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9411968/providers/microsoft.monitor/accounts/hcps-j9411968\", namespace=\"ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-w2fmt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T16:49:11Z\n Ended: 2026-10-13T17:06:14Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9411968-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.192:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9411968/providers/microsoft.monitor/accounts/hcps-j9411968\", namespace=\"ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-w2fmt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2te0njd9raq4236kje9rhni2tjsaju6i-s2p4w6o3n3b4d5j/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"ccf5e3543a16764030a181a888aebb366e2ffc4ccddbe613fa441fa1c60db349","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110027575299411968","occurred_at":"2026-10-13T15:15:38Z","failed_at":"e2e","signature_id":"5c9c535151da7212f6c377a91341a3089f208d25d42e32c59743276d5adb4a39","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:89]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0x34adff672b00\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-5xtjf6plnvzj\" to finish creating: GET https://rp.j9411968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a54bc6-22fe-435f-b062-cfb9be4ee7f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a54bc6-22fe-435f-b062-cfb9be4ee7f5\",\n \"name\": \"15a54bc6-22fe-435f-b062-cfb9be4ee7f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:18:20.725051746Z\",\n \"endTime\": \"2026-10-13T16:47:20.13643423Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9411968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a54bc6-22fe-435f-b062-cfb9be4ee7f5\n correlation ID: 1e394c4c-0ffa-4d52-86b7-718f2997f7ad\n client request ID: 573494d7-32aa-472d-997a-4bd8db38dacc\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:89]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0x34adff672b00\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-5xtjf6plnvzj\" to finish creating: GET https://rp.j9411968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a54bc6-22fe-435f-b062-cfb9be4ee7f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a54bc6-22fe-435f-b062-cfb9be4ee7f5\",\n \"name\": \"15a54bc6-22fe-435f-b062-cfb9be4ee7f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T16:18:20.725051746Z\",\n \"endTime\": \"2026-10-13T16:47:20.13643423Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9411968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a54bc6-22fe-435f-b062-cfb9be4ee7f5\n correlation ID: 1e394c4c-0ffa-4d52-86b7-718f2997f7ad\n client request ID: 573494d7-32aa-472d-997a-4bd8db38dacc\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110020079658733568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"closed","pr_sha":"630e96fc886b7febb635e87a2628c0dd14469dff","final_merged_sha":"d13cfaeca0f2297f74c1d9cb87d2b1f755473999","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T14:49:23Z","started_at":"2026-10-13T14:49:23Z","completed_at":"2026-10-13T17:15:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T17:24:52.345858479Z","timing_metadata_checked_at":"2026-10-13T17:24:52.345858479Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T17:24:52.349448368Z","region_metadata_checked_at":"2026-10-13T17:24:52.349448368Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fdb0a2ad845ed84fe846359f4f9968e1f8d5ccea06a1db9be38a77e992e79e41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110020079658733568","occurred_at":"2026-10-13T14:49:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110013226245689344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7480,"pr_state":"closed","pr_sha":"eb7fb7d5a1f82a8fe21f55a1c584a81677463ffa","final_merged_sha":"230a5d27c416299f628f0667ba08deeea65c4c88","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T14:22:09Z","started_at":"2026-10-13T14:22:09Z","completed_at":"2026-10-13T17:12:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T17:24:52.830236505Z","timing_metadata_checked_at":"2026-10-13T17:24:52.830236505Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T17:24:52.832944096Z","region_metadata_checked_at":"2026-10-13T17:24:52.832944096Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6e78ff738796b94121963e1d4f053a990842d44c3d9e0a5b81acefca34f0f851","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7480/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110013226245689344","occurred_at":"2026-10-13T14:22:09Z","failed_at":"e2e","signature_id":"45e5d5f08ac9de30d727c5c8b1bb049fe1a2cc352bd7a84c7804f7b87b200c41","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:139]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1ba7ee626c00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-jk8fzdt4b4dc\" to finish creating: GET https://rp.j5689344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8fa1502c-ea74-497a-b801-49d332ad2c8c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8fa1502c-ea74-497a-b801-49d332ad2c8c\",\n \"name\": \"8fa1502c-ea74-497a-b801-49d332ad2c8c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T15:47:39.824514687Z\",\n \"endTime\": \"2026-10-13T16:16:43.528270227Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-kv8bw-nb989: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-jk8fzdt4b4dc--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-kv8bw-nb989 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-kv8bw-nb989: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-jk8fzdt4b4dc--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-kv8bw-nb989 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5689344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8fa1502c-ea74-497a-b801-49d332ad2c8c\n correlation ID: 4064747a-e26a-4fb7-a7aa-4d63a3fd0e24\n client request ID: 6ee5609d-a8f6-47ef-be97-c1455bdfadcf\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:139]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1ba7ee626c00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-jk8fzdt4b4dc\" to finish creating: GET https://rp.j5689344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8fa1502c-ea74-497a-b801-49d332ad2c8c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8fa1502c-ea74-497a-b801-49d332ad2c8c\",\n \"name\": \"8fa1502c-ea74-497a-b801-49d332ad2c8c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T15:47:39.824514687Z\",\n \"endTime\": \"2026-10-13T16:16:43.528270227Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-kv8bw-nb989: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-jk8fzdt4b4dc--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-kv8bw-nb989 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-kv8bw-nb989: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-jk8fzdt4b4dc--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-kv8bw-nb989 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5689344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8fa1502c-ea74-497a-b801-49d332ad2c8c\n correlation ID: 4064747a-e26a-4fb7-a7aa-4d63a3fd0e24\n client request ID: 6ee5609d-a8f6-47ef-be97-c1455bdfadcf","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dpn7kns9h5s6\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110012110980255744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7493,"pr_state":"open","pr_sha":"7c83ae5457b3a086bef52e98073272d484d30759","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T14:17:26Z","started_at":"2026-10-13T14:17:26Z","completed_at":"2026-10-13T17:12:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T17:24:53.211840565Z","timing_metadata_checked_at":"2026-10-13T17:24:53.211840565Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T17:24:53.214409856Z","region_metadata_checked_at":"2026-10-13T17:24:53.214409856Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110003822465126400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7397,"pr_state":"closed","pr_sha":"88f953df45eba2a91a51c0ad97acee3993bb57a9","final_merged_sha":"88f953df45eba2a91a51c0ad97acee3993bb57a9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T13:44:47Z","started_at":"2026-10-13T13:44:47Z","completed_at":"2026-10-13T15:17:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T15:28:34.428594121Z","timing_metadata_checked_at":"2026-10-13T15:28:34.428594121Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T15:28:34.432274109Z","region_metadata_checked_at":"2026-10-13T15:28:34.432274109Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"bf67032da7001728f6e5bef1b1689cbf853c37c66f54518432be2d5dc65613db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110003822465126400","occurred_at":"2026-10-13T13:44:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110002534083989504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7493,"pr_state":"open","pr_sha":"7c83ae5457b3a086bef52e98073272d484d30759","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T13:39:40Z","started_at":"2026-10-13T13:39:40Z","completed_at":"2026-10-13T14:17:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T15:18:10.291402438Z","timing_metadata_checked_at":"2026-10-13T15:18:10.291402438Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T15:18:10.29381713Z","region_metadata_checked_at":"2026-10-13T15:18:10.29381713Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5846754116fa540a985e938f21502eb729d78596af5da8889ad991a6964ec708","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7493/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110002534083989504","occurred_at":"2026-10-13T13:39:40Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7370/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110001541002825728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7370,"pr_state":"open","pr_sha":"b9c440a0998e4ec108f167cfea60ab086f4c315f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T13:35:28Z","started_at":"2026-10-13T13:35:28Z","completed_at":"2026-10-13T16:03:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T16:10:49.374918433Z","timing_metadata_checked_at":"2026-10-13T16:10:49.374918433Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T16:10:49.378693621Z","region_metadata_checked_at":"2026-10-13T16:10:49.378693621Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7492/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2110000696223862784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7492,"pr_state":"closed","pr_sha":"5bf7c6eb96af486e0a3ec4a5df97a32a7e503808","final_merged_sha":"5bf7c6eb96af486e0a3ec4a5df97a32a7e503808","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-13T13:32:22Z","started_at":"2026-10-13T13:32:22Z","completed_at":"2026-10-13T15:57:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T16:10:49.552562767Z","timing_metadata_checked_at":"2026-10-13T16:10:49.552562767Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T16:10:49.555693857Z","region_metadata_checked_at":"2026-10-13T16:10:49.555693857Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109994317174018048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7488,"pr_state":"closed","pr_sha":"e343803f365d228cc3e0070ed414c588a26b5e9f","final_merged_sha":"e343803f365d228cc3e0070ed414c588a26b5e9f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T13:07:01Z","started_at":"2026-10-13T13:07:01Z","completed_at":"2026-10-13T15:24:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T15:39:15.195975696Z","timing_metadata_checked_at":"2026-10-13T15:39:15.195975696Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T15:39:15.199372384Z","region_metadata_checked_at":"2026-10-13T15:39:15.199372384Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5512a469a8692703fecdc26b1a1e08bef4b2dc230cd2b519ed4eee9d11efba54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109994317174018048","occurred_at":"2026-10-13T13:07:01Z","failed_at":"e2e","signature_id":"64c28e7e5c45c72d72e63cae7d8453adeb4e0d6a5ad577f87fac746a2ba5e16b","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x28b03e9a6680\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-69pgmlxbh7vh\" to finish creating: GET https://rp.j4018048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\",\n \"name\": \"b2e67201-af0c-42da-b508-f40f254f87c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T14:11:58.76773642Z\",\n \"endTime\": \"2026-10-13T14:41:02.859999025Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-69pgmlxbh7vh--managed/providers/Microsoft.Compute/virtualMachines/d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-69pgmlxbh7vh--managed/providers/Microsoft.Compute/virtualMachines/d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4018048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\n correlation ID: c7972c9d-9ddb-4652-a9ac-8fbcdb522c95\n client request ID: e9e05827-93d6-4587-bf66-ed9dc69c4222\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x28b03e9a6680\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-69pgmlxbh7vh\" to finish creating: GET https://rp.j4018048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\",\n \"name\": \"b2e67201-af0c-42da-b508-f40f254f87c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T14:11:58.76773642Z\",\n \"endTime\": \"2026-10-13T14:41:02.859999025Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-69pgmlxbh7vh--managed/providers/Microsoft.Compute/virtualMachines/d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-69pgmlxbh7vh--managed/providers/Microsoft.Compute/virtualMachines/d0b5y3p4k9a2p9w-np-1-wtm4z-m5q6d to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4018048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b2e67201-af0c-42da-b508-f40f254f87c7\n correlation ID: c7972c9d-9ddb-4652-a9ac-8fbcdb522c95\n client request ID: e9e05827-93d6-4587-bf66-ed9dc69c4222\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b999807f7a72c8fd068cca967eb322c5413754f2f246f0b6b4c439eff9f797fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109994317174018048","occurred_at":"2026-10-13T13:07:01Z","failed_at":"alert","signature_id":"84707bded3f4fb605d70b06f7b06b6aaa5f660163675ed498765d69734732aa9","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T15:08:37Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4018048-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.52:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4018048/providers/microsoft.monitor/accounts/services-j4018048\", namespace=\"klusterlet-2tdunktpkavtvflvbc7rdslt6p35hl1d\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-kq6qh\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tdunktpkavtvflvbc7rdslt6p35hl1d on cluster ci01-j4018048-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109989963910090752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7477,"pr_state":"closed","pr_sha":"c8eaee8afc8ecc8c3ca6bbb23d5630f981cf07cb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T12:49:43Z","started_at":"2026-10-13T12:49:43Z","completed_at":"2026-10-13T13:06:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T13:21:32.554518686Z","timing_metadata_checked_at":"2026-10-13T13:21:32.554518686Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-13T13:21:32.557884977Z","region_metadata_checked_at":"2026-10-13T14:24:51.39785835Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"56d32f8744ef583b8d7380d0712375f6130ece24d13782f897a50ff2cc3f109a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7477/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109989963910090752","occurred_at":"2026-10-13T12:49:43Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109985456534851584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7441,"pr_state":"open","pr_sha":"3148140c1e79833ec6ffb3fd5c8e53e5bc5b160f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T12:31:49Z","started_at":"2026-10-13T12:31:49Z","completed_at":"2026-10-13T14:48:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T15:07:41.209098202Z","timing_metadata_checked_at":"2026-10-13T15:07:41.209098202Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T15:07:41.214209587Z","region_metadata_checked_at":"2026-10-13T15:07:41.214209587Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ec9cdfa0c4220928ec1888ba0518489b8029bcea7bd08b8718b6f0d7f6341abd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109985456534851584","occurred_at":"2026-10-13T12:31:49Z","failed_at":"alert","signature_id":"e0d8a21444c75df32d1ee0ca0a5977901884526027e6eba129d7da65137483ea","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T14:30:35Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j4851584-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4851584/providers/microsoft.monitor/accounts/services-j4851584\", namespace=\"klusterlet-2tdu5qet089m4vq4jt27iijio79srvjo\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-fgb47\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tdu5qet089m4vq4jt27iijio79srvjo on cluster ci01-j4851584-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109983366630608896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7469,"pr_state":"closed","pr_sha":"9e5632fe32eeeb715ec812db0dc2cac204cd1087","final_merged_sha":"a2d7fd940b1ba037c80cee86cf5b380e84ab4193","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T12:23:30Z","started_at":"2026-10-13T12:23:30Z","completed_at":"2026-10-13T14:06:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T15:18:10.271643205Z","timing_metadata_checked_at":"2026-10-13T15:18:10.271643205Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T15:18:10.274132897Z","region_metadata_checked_at":"2026-10-13T15:18:10.274132897Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"10142cf13c401bea3227cb7747dc25e12067c1a2e6111b05a9555e800c5853a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7469/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109983366630608896","occurred_at":"2026-10-13T12:23:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109978799176159232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"f9f9f546696bce226d7b2c56078c6afa008c2089","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T12:05:21Z","started_at":"2026-10-13T12:05:21Z","completed_at":"2026-10-13T14:23:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T14:35:13.599704412Z","timing_metadata_checked_at":"2026-10-13T14:35:13.599704412Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T14:35:13.602871602Z","region_metadata_checked_at":"2026-10-13T14:35:13.602871602Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fb3b0a07ba2b2e1a1bc780352d294aa6c05919acf2b9a1fe40514c9b2b7a9166","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109978799176159232","occurred_at":"2026-10-13T12:05:21Z","failed_at":"e2e","signature_id":"0f4249b820a1949de5b8be0a7d0f2992e157ed0020fa38fe47a583da35703e09","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3398cf888a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-qzrlv9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Authorization/roleAssignments/adc91f81-bb61-5a2b-b4cc-0e8c1c460b92' at line '1' and column '27135'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n \\\"additionalInfo\\\": [\\r\\n {\\r\\n \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n \\\"info\\\": {\\r\\n \\\"lineNumber\\\": 1,\\r\\n \\\"linePosition\\\": 27135,\\r\\n \\\"path\\\": \\\"\\\"\\r\\n }\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n correlation ID: 9860797c-226a-442a-b3bd-c7b7c62b766b\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x3398cf888a80\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-autoscaling-hcp-cluster-qzrlv9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-e4ixwyxsfjnje\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cml7gjf827gp/providers/Microsoft.Authorization/roleAssignments/adc91f81-bb61-5a2b-b4cc-0e8c1c460b92' at line '1' and column '27135'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n \\\"additionalInfo\\\": [\\r\\n {\\r\\n \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n \\\"info\\\": {\\r\\n \\\"lineNumber\\\": 1,\\r\\n \\\"linePosition\\\": 27135,\\r\\n \\\"path\\\": \\\"\\\"\\r\\n }\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-autoscaling-hcp-cluster-qzrlv9/operationStatuses/08584097109520399682\n correlation ID: 9860797c-226a-442a-b3bd-c7b7c62b766b\n client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d7dfb9e704f567d65cc7ccb0bddcc27f123f1af0b24cca84fa1042281e3a756c","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InvalidTemplate; detail message Unable to process template language expressions for resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Authorization/roleAssignments/\u003cuuid\u003e' at line '\u003cline\u003e' and column '\u003ccolumn\u003e'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d7dfb9e704f567d65cc7ccb0bddcc27f123f1af0b24cca84fa1042281e3a756c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7463,"pr_state":"open","pr_sha":"d7962714a9b15f32d56ed0cc4ccd1046008018fb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T11:58:43Z","started_at":"2026-10-13T11:58:43Z","completed_at":"2026-10-13T14:23:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T14:35:13.789666704Z","timing_metadata_checked_at":"2026-10-13T14:35:13.789666704Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T14:35:13.792422495Z","region_metadata_checked_at":"2026-10-13T14:35:13.792422495Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"19f4a7734386c1abbaf4d2aa95e1196892b3e6d129ac36e3ffa4630d12e1d4ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","failed_at":"alert","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T13:04:07Z\n Ended: 2026-10-13T13:24:08Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9120896-svc\", component=\"backend\", name=\"actualhostedcluster\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue actualhostedcluster has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"25e9a00a67476564c3ca1649851da2a15222724b2d7e41beda60ec0399599aeb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","failed_at":"e2e","signature_id":"bd20bcea67709cdf0163960d867a7fb6af562bc860fdce2d4fddc185e5ee2f10","test_name":"Verify cluster operators should provision an HCP cluster and verify the cluster operators","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_operators.go:163]: failed to verify cluster operators on cluster \"cluster-operators-qwqhtt\"\nUnexpected error:\n \u003c*errors.joinError | 0x275bb568108\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet assigned to a node\n {\n errs: [\n \u003c*fmt.wrapError | 0x275bb4fe480\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*fmt.wrapError | 0x275baa78f20\u003e{\n msg: \"EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*fmt.wrapError | 0x275baa78d80\u003e{\n msg: \"timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*errors.errorString | 0x275bb7584c0\u003e{\n s: \"EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_operators.go:163]: failed to verify cluster operators on cluster \"cluster-operators-qwqhtt\"\nUnexpected error:\n \u003c*errors.joinError | 0x275bb568108\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet assigned to a node\n {\n errs: [\n \u003c*fmt.wrapError | 0x275bb4fe480\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*fmt.wrapError | 0x275baa78f20\u003e{\n msg: \"EgressIP \\\"e2e-cloud-network-config-test\\\" was never assigned to a node: timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*fmt.wrapError | 0x275baa78d80\u003e{\n msg: \"timed out waiting for EgressIP \\\"e2e-cloud-network-config-test\\\" to be assigned to a node: EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n err: \u003c*errors.errorString | 0x275bb7584c0\u003e{\n s: \"EgressIP \\\"e2e-cloud-network-config-test\\\" is not yet assigned to a node\",\n },\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35f4b32d0c9a217f6e6dcfd9cb816a96a3bf1d26a1497dfd78708f5764de5a24","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.6) failed: EgressIP \"e2e-cloud-network-config-test\" was never assigned to a node: timed out waiting for EgressIP \"e2e-cloud-network-config-test\" to be assigned to a node: EgressIP \"e2e-cloud-network-config-test\" is not yet assigned to a node"}},{"row_id":"2bfcbdaf3097fbc2cda18fb8e4d7c324a2386109fc688ea082bf9ef4413918e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","failed_at":"alert","signature_id":"b0c665dc5593bf939be3f561a3b7d6c563fa192fba4413e100dc7bc2db835e54","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 268 time(s) (214 unknown, 54 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:24:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-4p8nw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-4p8nw has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:21:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-mth8b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-mth8b has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-kfrcc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-kfrcc has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:21:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-6mmc5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-6mmc5 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-gvsss\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-gvsss has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-cvw52\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-cvw52 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-64j6b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-64j6b has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:21:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-fk5mc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-fk5mc has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:21:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-htwvx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-htwvx has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-xzksr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-xzksr has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:21:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-xd6hs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-xd6hs has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-qwtqs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-qwtqs has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-cd628\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-cd628 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:24:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-5lhgf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-5lhgf has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-pgq56\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-pgq56 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:24:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-hvd9c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-hvd9c has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-v8qst\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-v8qst has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-mslbg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-mslbg has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-fpk2l\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-fpk2l has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:21:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-rfn8f\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-rfn8f has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-nm4ns\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-nm4ns has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-9vjkc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-9vjkc has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-krtvh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-krtvh has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-ckztq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-ckztq has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:24:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-jp2dp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-jp2dp has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-w6nlz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-w6nlz has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-13T13:07:29Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-xlqmw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-xlqmw has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-84j5z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-84j5z has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-mpcxc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-mpcxc has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-llsls\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-llsls has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-2z42c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-2z42c has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-42tsp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-42tsp has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-5w88x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-5w88x has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:24:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-mmcn9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-mmcn9 has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-w8l2m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-w8l2m has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-c9bzq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-c9bzq has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:24:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-dqskl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-dqskl has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:24:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-2244t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-2244t has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-wjptp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-wjptp has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-tjhrj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-tjhrj has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:21:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-646s4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-646s4 has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:21:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-hn9r4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-hn9r4 has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-kssd5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-kssd5 has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:24:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-2wxwt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-2wxwt has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-5ffbw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-5ffbw has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-zszss\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-zszss has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-p2jgh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-p2jgh has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-13T13:07:30Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-8fgnv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-8fgnv has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-13T13:07:31Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-h9gpx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-h9gpx has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-13T13:07:31Z\n Ended: 2026-10-13T13:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-9dt95\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-9dt95 has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-13T13:07:31Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-ntx2r\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-ntx2r has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-13T13:07:31Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-hvjt4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-hvjt4 has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-13T13:07:31Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-b9h4w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-b9h4w has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-13T13:07:31Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-qzr2z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-qzr2z has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-10-13T13:07:31Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-zh7v8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-zh7v8 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-zxnfn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-zxnfn has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-ccvlz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-ccvlz has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-5sffd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-5sffd has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-sn9tj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-sn9tj has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:21:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-kv44x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-kv44x has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-5jlm5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-5jlm5 has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-k8g2p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-k8g2p has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:21:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-dx2bl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-dx2bl has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-7jxb4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-7jxb4 has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-nfwzr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-nfwzr has been in a non-ready state for longer than 5 minutes.\nFiring 66:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-9vn2r\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-9vn2r has been in a non-ready state for longer than 5 minutes.\nFiring 67:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-fp94v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-fp94v has been in a non-ready state for longer than 5 minutes.\nFiring 68:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:21:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-7tgzd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-7tgzd has been in a non-ready state for longer than 5 minutes.\nFiring 69:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-rj5rm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-rj5rm has been in a non-ready state for longer than 5 minutes.\nFiring 70:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-4vxxx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-4vxxx has been in a non-ready state for longer than 5 minutes.\nFiring 71:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-xd6zz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-xd6zz has been in a non-ready state for longer than 5 minutes.\nFiring 72:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:24:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-kpw8t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-kpw8t has been in a non-ready state for longer than 5 minutes.\nFiring 73:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-8kw4b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-8kw4b has been in a non-ready state for longer than 5 minutes.\nFiring 74:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-nrhmx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-nrhmx has been in a non-ready state for longer than 5 minutes.\nFiring 75:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-jz6bl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-jz6bl has been in a non-ready state for longer than 5 minutes.\nFiring 76:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:24:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-mqw5p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-mqw5p has been in a non-ready state for longer than 5 minutes.\nFiring 77:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-rbmn4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-rbmn4 has been in a non-ready state for longer than 5 minutes.\nFiring 78:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-tf5n4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-tf5n4 has been in a non-ready state for longer than 5 minutes.\nFiring 79:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:21:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-khsng\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-khsng has been in a non-ready state for longer than 5 minutes.\nFiring 80:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-8tmzd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-8tmzd has been in a non-ready state for longer than 5 minutes.\nFiring 81:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-5skgc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-5skgc has been in a non-ready state for longer than 5 minutes.\nFiring 82:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-59ktc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-59ktc has been in a non-ready state for longer than 5 minutes.\nFiring 83:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-z9qk6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-z9qk6 has been in a non-ready state for longer than 5 minutes.\nFiring 84:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:24:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-6wcbs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-6wcbs has been in a non-ready state for longer than 5 minutes.\nFiring 85:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-2vc6m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-2vc6m has been in a non-ready state for longer than 5 minutes.\nFiring 86:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-4xw4t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-4xw4t has been in a non-ready state for longer than 5 minutes.\nFiring 87:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-88nm4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-88nm4 has been in a non-ready state for longer than 5 minutes.\nFiring 88:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:24:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-v8bjr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-v8bjr has been in a non-ready state for longer than 5 minutes.\nFiring 89:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:21:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-6p6dh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-6p6dh has been in a non-ready state for longer than 5 minutes.\nFiring 90:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-fmbv8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-fmbv8 has been in a non-ready state for longer than 5 minutes.\nFiring 91:\n State: Resolved\n Started: 2026-10-13T13:07:32Z\n Ended: 2026-10-13T13:24:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-hrnst\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-hrnst has been in a non-ready state for longer than 5 minutes.\nFiring 92:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-cnb9k\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-cnb9k has been in a non-ready state for longer than 5 minutes.\nFiring 93:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-hrqjl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-hrqjl has been in a non-ready state for longer than 5 minutes.\nFiring 94:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-b4llx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-b4llx has been in a non-ready state for longer than 5 minutes.\nFiring 95:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-d65nj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-d65nj has been in a non-ready state for longer than 5 minutes.\nFiring 96:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-kfpll\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-kfpll has been in a non-ready state for longer than 5 minutes.\nFiring 97:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-rr42p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-rr42p has been in a non-ready state for longer than 5 minutes.\nFiring 98:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-kskk4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-kskk4 has been in a non-ready state for longer than 5 minutes.\nFiring 99:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-rwcpc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-rwcpc has been in a non-ready state for longer than 5 minutes.\nFiring 100:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:24:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-4vtl9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-4vtl9 has been in a non-ready state for longer than 5 minutes.\nFiring 101:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-vfdsj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-vfdsj has been in a non-ready state for longer than 5 minutes.\nFiring 102:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:24:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-zdlpc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-zdlpc has been in a non-ready state for longer than 5 minutes.\nFiring 103:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-22pkh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-22pkh has been in a non-ready state for longer than 5 minutes.\nFiring 104:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-b7v7h\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-b7v7h has been in a non-ready state for longer than 5 minutes.\nFiring 105:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-4dtxs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-4dtxs has been in a non-ready state for longer than 5 minutes.\nFiring 106:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-lqsrv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-lqsrv has been in a non-ready state for longer than 5 minutes.\nFiring 107:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:21:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-5kmv8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-5kmv8 has been in a non-ready state for longer than 5 minutes.\nFiring 108:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-rdfmh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-rdfmh has been in a non-ready state for longer than 5 minutes.\nFiring 109:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:21:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-dr2lh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-dr2lh has been in a non-ready state for longer than 5 minutes.\nFiring 110:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:21:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-p685x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-p685x has been in a non-ready state for longer than 5 minutes.\nFiring 111:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-dw2mp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-dw2mp has been in a non-ready state for longer than 5 minutes.\nFiring 112:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-657fn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-657fn has been in a non-ready state for longer than 5 minutes.\nFiring 113:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-jfq9s\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-jfq9s has been in a non-ready state for longer than 5 minutes.\nFiring 114:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-k4tcm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-k4tcm has been in a non-ready state for longer than 5 minutes.\nFiring 115:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:24:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-t8bkn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-t8bkn has been in a non-ready state for longer than 5 minutes.\nFiring 116:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-tx7lr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-tx7lr has been in a non-ready state for longer than 5 minutes.\nFiring 117:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:24:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-6j7bp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-6j7bp has been in a non-ready state for longer than 5 minutes.\nFiring 118:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-rx2xw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-rx2xw has been in a non-ready state for longer than 5 minutes.\nFiring 119:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-5ljqm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-5ljqm has been in a non-ready state for longer than 5 minutes.\nFiring 120:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-2tfvd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-2tfvd has been in a non-ready state for longer than 5 minutes.\nFiring 121:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-8n2np\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-8n2np has been in a non-ready state for longer than 5 minutes.\nFiring 122:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:21:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-cfdxv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-cfdxv has been in a non-ready state for longer than 5 minutes.\nFiring 123:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-bprfj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-bprfj has been in a non-ready state for longer than 5 minutes.\nFiring 124:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-hnr8c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-hnr8c has been in a non-ready state for longer than 5 minutes.\nFiring 125:\n State: Resolved\n Started: 2026-10-13T13:07:33Z\n Ended: 2026-10-13T13:21:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-l9z76\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-l9z76 has been in a non-ready state for longer than 5 minutes.\nFiring 126:\n State: Resolved\n Started: 2026-10-13T13:07:34Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-r9dl8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-r9dl8 has been in a non-ready state for longer than 5 minutes.\nFiring 127:\n State: Resolved\n Started: 2026-10-13T13:07:34Z\n Ended: 2026-10-13T13:21:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-8sjtl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-8sjtl has been in a non-ready state for longer than 5 minutes.\nFiring 128:\n State: Resolved\n Started: 2026-10-13T13:07:34Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-hmhnl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-hmhnl has been in a non-ready state for longer than 5 minutes.\nFiring 129:\n State: Resolved\n Started: 2026-10-13T13:07:34Z\n Ended: 2026-10-13T13:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-v79bm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-v79bm has been in a non-ready state for longer than 5 minutes.\nFiring 130:\n State: Resolved\n Started: 2026-10-13T13:07:34Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-zwghk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-zwghk has been in a non-ready state for longer than 5 minutes.\nFiring 131:\n State: Resolved\n Started: 2026-10-13T13:07:34Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-fj72b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-fj72b has been in a non-ready state for longer than 5 minutes.\nFiring 132:\n State: Resolved\n Started: 2026-10-13T13:07:34Z\n Ended: 2026-10-13T13:21:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-ppzvj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-ppzvj has been in a non-ready state for longer than 5 minutes.\nFiring 133:\n State: Resolved\n Started: 2026-10-13T13:07:34Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-jvdj2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-jvdj2 has been in a non-ready state for longer than 5 minutes.\nFiring 134:\n State: Resolved\n Started: 2026-10-13T13:07:35Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-fp2ch\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-fp2ch has been in a non-ready state for longer than 5 minutes.\nFiring 135:\n State: Resolved\n Started: 2026-10-13T13:07:35Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-gbcsh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-gbcsh has been in a non-ready state for longer than 5 minutes.\nFiring 136:\n State: Resolved\n Started: 2026-10-13T13:07:35Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-7lpps\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-7lpps has been in a non-ready state for longer than 5 minutes.\nFiring 137:\n State: Resolved\n Started: 2026-10-13T13:07:35Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-dprms\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-dprms has been in a non-ready state for longer than 5 minutes.\nFiring 138:\n State: Resolved\n Started: 2026-10-13T13:07:35Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-9w7bz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-9w7bz has been in a non-ready state for longer than 5 minutes.\nFiring 139:\n State: Resolved\n Started: 2026-10-13T13:07:35Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-45l4z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-45l4z has been in a non-ready state for longer than 5 minutes.\nFiring 140:\n State: Resolved\n Started: 2026-10-13T13:07:36Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-s625c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-s625c has been in a non-ready state for longer than 5 minutes.\nFiring 141:\n State: Resolved\n Started: 2026-10-13T13:07:36Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-629qx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-629qx has been in a non-ready state for longer than 5 minutes.\nFiring 142:\n State: Resolved\n Started: 2026-10-13T13:07:36Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-qb6nb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-qb6nb has been in a non-ready state for longer than 5 minutes.\nFiring 143:\n State: Resolved\n Started: 2026-10-13T13:07:36Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-7nmbz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-7nmbz has been in a non-ready state for longer than 5 minutes.\nFiring 144:\n State: Resolved\n Started: 2026-10-13T13:07:36Z\n Ended: 2026-10-13T13:24:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-pthc4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-pthc4 has been in a non-ready state for longer than 5 minutes.\nFiring 145:\n State: Resolved\n Started: 2026-10-13T13:07:36Z\n Ended: 2026-10-13T13:24:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-v8p7j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-v8p7j has been in a non-ready state for longer than 5 minutes.\nFiring 146:\n State: Resolved\n Started: 2026-10-13T13:07:36Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-xzw8c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-xzw8c has been in a non-ready state for longer than 5 minutes.\nFiring 147:\n State: Resolved\n Started: 2026-10-13T13:07:37Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-lchx8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-lchx8 has been in a non-ready state for longer than 5 minutes.\nFiring 148:\n State: Resolved\n Started: 2026-10-13T13:07:37Z\n Ended: 2026-10-13T13:24:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-vpjc5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-vpjc5 has been in a non-ready state for longer than 5 minutes.\nFiring 149:\n State: Resolved\n Started: 2026-10-13T13:07:37Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-xxtjb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-xxtjb has been in a non-ready state for longer than 5 minutes.\nFiring 150:\n State: Resolved\n Started: 2026-10-13T13:07:37Z\n Ended: 2026-10-13T13:21:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-k8d2n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-k8d2n has been in a non-ready state for longer than 5 minutes.\nFiring 151:\n State: Resolved\n Started: 2026-10-13T13:07:37Z\n Ended: 2026-10-13T13:24:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-gxl6f\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-gxl6f has been in a non-ready state for longer than 5 minutes.\nFiring 152:\n State: Resolved\n Started: 2026-10-13T13:07:37Z\n Ended: 2026-10-13T13:21:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-86q5x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-86q5x has been in a non-ready state for longer than 5 minutes.\nFiring 153:\n State: Resolved\n Started: 2026-10-13T13:07:37Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-gj2jg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-gj2jg has been in a non-ready state for longer than 5 minutes.\nFiring 154:\n State: Resolved\n Started: 2026-10-13T13:07:37Z\n Ended: 2026-10-13T13:25:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-f2jnj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-f2jnj has been in a non-ready state for longer than 5 minutes.\nFiring 155:\n State: Resolved\n Started: 2026-10-13T13:07:37Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-4kt8w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-4kt8w has been in a non-ready state for longer than 5 minutes.\nFiring 156:\n State: Resolved\n Started: 2026-10-13T13:07:37Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-2rcrh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-2rcrh has been in a non-ready state for longer than 5 minutes.\nFiring 157:\n State: Resolved\n Started: 2026-10-13T13:07:38Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-cml42\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-cml42 has been in a non-ready state for longer than 5 minutes.\nFiring 158:\n State: Resolved\n Started: 2026-10-13T13:07:38Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-6lvzr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-6lvzr has been in a non-ready state for longer than 5 minutes.\nFiring 159:\n State: Resolved\n Started: 2026-10-13T13:07:38Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-hrjz4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-hrjz4 has been in a non-ready state for longer than 5 minutes.\nFiring 160:\n State: Resolved\n Started: 2026-10-13T13:07:38Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-zs9rn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-zs9rn has been in a non-ready state for longer than 5 minutes.\nFiring 161:\n State: Resolved\n Started: 2026-10-13T13:07:38Z\n Ended: 2026-10-13T13:21:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-4bkzc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-4bkzc has been in a non-ready state for longer than 5 minutes.\nFiring 162:\n State: Resolved\n Started: 2026-10-13T13:07:38Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-s2mfz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-s2mfz has been in a non-ready state for longer than 5 minutes.\nFiring 163:\n State: Resolved\n Started: 2026-10-13T13:07:38Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-qqgnn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-qqgnn has been in a non-ready state for longer than 5 minutes.\nFiring 164:\n State: Resolved\n Started: 2026-10-13T13:07:39Z\n Ended: 2026-10-13T13:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-frbjb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-frbjb has been in a non-ready state for longer than 5 minutes.\nFiring 165:\n State: Resolved\n Started: 2026-10-13T13:07:39Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-h78dn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-h78dn has been in a non-ready state for longer than 5 minutes.\nFiring 166:\n State: Resolved\n Started: 2026-10-13T13:07:39Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-wfxjs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-wfxjs has been in a non-ready state for longer than 5 minutes.\nFiring 167:\n State: Resolved\n Started: 2026-10-13T13:07:39Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-9n978\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-9n978 has been in a non-ready state for longer than 5 minutes.\nFiring 168:\n State: Resolved\n Started: 2026-10-13T13:07:39Z\n Ended: 2026-10-13T13:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-6k44h\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-6k44h has been in a non-ready state for longer than 5 minutes.\nFiring 169:\n State: Resolved\n Started: 2026-10-13T13:07:40Z\n Ended: 2026-10-13T13:24:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-dr7bd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-dr7bd has been in a non-ready state for longer than 5 minutes.\nFiring 170:\n State: Resolved\n Started: 2026-10-13T13:07:40Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-sp5tk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-sp5tk has been in a non-ready state for longer than 5 minutes.\nFiring 171:\n State: Resolved\n Started: 2026-10-13T13:07:40Z\n Ended: 2026-10-13T13:21:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-lrjdn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-lrjdn has been in a non-ready state for longer than 5 minutes.\nFiring 172:\n State: Resolved\n Started: 2026-10-13T13:07:40Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-ffgrn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-ffgrn has been in a non-ready state for longer than 5 minutes.\nFiring 173:\n State: Resolved\n Started: 2026-10-13T13:07:40Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-5jvql\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-5jvql has been in a non-ready state for longer than 5 minutes.\nFiring 174:\n State: Resolved\n Started: 2026-10-13T13:07:40Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-2gt9h\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-2gt9h has been in a non-ready state for longer than 5 minutes.\nFiring 175:\n State: Resolved\n Started: 2026-10-13T13:07:40Z\n Ended: 2026-10-13T13:21:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-tgwzt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-tgwzt has been in a non-ready state for longer than 5 minutes.\nFiring 176:\n State: Resolved\n Started: 2026-10-13T13:07:40Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-lmzwn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-lmzwn has been in a non-ready state for longer than 5 minutes.\nFiring 177:\n State: Resolved\n Started: 2026-10-13T13:07:40Z\n Ended: 2026-10-13T13:24:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-dtbpb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-dtbpb has been in a non-ready state for longer than 5 minutes.\nFiring 178:\n State: Resolved\n Started: 2026-10-13T13:07:41Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-gkljq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-gkljq has been in a non-ready state for longer than 5 minutes.\nFiring 179:\n State: Resolved\n Started: 2026-10-13T13:07:41Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-v6w29\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-v6w29 has been in a non-ready state for longer than 5 minutes.\nFiring 180:\n State: Resolved\n Started: 2026-10-13T13:07:41Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-7drh8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-7drh8 has been in a non-ready state for longer than 5 minutes.\nFiring 181:\n State: Resolved\n Started: 2026-10-13T13:07:41Z\n Ended: 2026-10-13T13:21:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-vqlm7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-vqlm7 has been in a non-ready state for longer than 5 minutes.\nFiring 182:\n State: Resolved\n Started: 2026-10-13T13:07:41Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-hqbqc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-hqbqc has been in a non-ready state for longer than 5 minutes.\nFiring 183:\n State: Resolved\n Started: 2026-10-13T13:07:41Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-v4cx8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-v4cx8 has been in a non-ready state for longer than 5 minutes.\nFiring 184:\n State: Resolved\n Started: 2026-10-13T13:07:41Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-v7rhp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-v7rhp has been in a non-ready state for longer than 5 minutes.\nFiring 185:\n State: Resolved\n Started: 2026-10-13T13:07:41Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-twpt9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-twpt9 has been in a non-ready state for longer than 5 minutes.\nFiring 186:\n State: Resolved\n Started: 2026-10-13T13:07:41Z\n Ended: 2026-10-13T13:21:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-lbp46\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-lbp46 has been in a non-ready state for longer than 5 minutes.\nFiring 187:\n State: Resolved\n Started: 2026-10-13T13:07:42Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-fvh9x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-fvh9x has been in a non-ready state for longer than 5 minutes.\nFiring 188:\n State: Resolved\n Started: 2026-10-13T13:07:42Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-7d2tq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-7d2tq has been in a non-ready state for longer than 5 minutes.\nFiring 189:\n State: Resolved\n Started: 2026-10-13T13:07:42Z\n Ended: 2026-10-13T13:24:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-k66gb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-k66gb has been in a non-ready state for longer than 5 minutes.\nFiring 190:\n State: Resolved\n Started: 2026-10-13T13:07:42Z\n Ended: 2026-10-13T13:21:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-x97ps\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-x97ps has been in a non-ready state for longer than 5 minutes.\nFiring 191:\n State: Resolved\n Started: 2026-10-13T13:07:42Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-9v7v5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-9v7v5 has been in a non-ready state for longer than 5 minutes.\nFiring 192:\n State: Resolved\n Started: 2026-10-13T13:07:42Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-l8wfz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-l8wfz has been in a non-ready state for longer than 5 minutes.\nFiring 193:\n State: Resolved\n Started: 2026-10-13T13:07:42Z\n Ended: 2026-10-13T13:21:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-jwnn7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-jwnn7 has been in a non-ready state for longer than 5 minutes.\nFiring 194:\n State: Resolved\n Started: 2026-10-13T13:07:42Z\n Ended: 2026-10-13T13:25:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-hp68j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-hp68j has been in a non-ready state for longer than 5 minutes.\nFiring 195:\n State: Resolved\n Started: 2026-10-13T13:07:42Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-f2rpx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-f2rpx has been in a non-ready state for longer than 5 minutes.\nFiring 196:\n State: Resolved\n Started: 2026-10-13T13:07:43Z\n Ended: 2026-10-13T13:24:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-2j9gr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-2j9gr has been in a non-ready state for longer than 5 minutes.\nFiring 197:\n State: Resolved\n Started: 2026-10-13T13:07:43Z\n Ended: 2026-10-13T13:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-rbfss\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-rbfss has been in a non-ready state for longer than 5 minutes.\nFiring 198:\n State: Resolved\n Started: 2026-10-13T13:07:43Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-lp2nr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-lp2nr has been in a non-ready state for longer than 5 minutes.\nFiring 199:\n State: Resolved\n Started: 2026-10-13T13:07:43Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-wcvm5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-wcvm5 has been in a non-ready state for longer than 5 minutes.\nFiring 200:\n State: Resolved\n Started: 2026-10-13T13:07:43Z\n Ended: 2026-10-13T13:24:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-pb78h\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-pb78h has been in a non-ready state for longer than 5 minutes.\nFiring 201:\n State: Resolved\n Started: 2026-10-13T13:07:44Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-v7mnn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-v7mnn has been in a non-ready state for longer than 5 minutes.\nFiring 202:\n State: Resolved\n Started: 2026-10-13T13:07:44Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-6pk5z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-6pk5z has been in a non-ready state for longer than 5 minutes.\nFiring 203:\n State: Resolved\n Started: 2026-10-13T13:07:44Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-b468t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-b468t has been in a non-ready state for longer than 5 minutes.\nFiring 204:\n State: Resolved\n Started: 2026-10-13T13:07:44Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-pd5l9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-pd5l9 has been in a non-ready state for longer than 5 minutes.\nFiring 205:\n State: Resolved\n Started: 2026-10-13T13:07:44Z\n Ended: 2026-10-13T13:25:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-8fhdq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-8fhdq has been in a non-ready state for longer than 5 minutes.\nFiring 206:\n State: Resolved\n Started: 2026-10-13T13:07:44Z\n Ended: 2026-10-13T13:25:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-fmjj6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-fmjj6 has been in a non-ready state for longer than 5 minutes.\nFiring 207:\n State: Resolved\n Started: 2026-10-13T13:07:44Z\n Ended: 2026-10-13T13:24:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-w96fn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-w96fn has been in a non-ready state for longer than 5 minutes.\nFiring 208:\n State: Resolved\n Started: 2026-10-13T13:07:44Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"network-node-identity-7ff6c58d49-29m5r\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/network-node-identity-7ff6c58d49-29m5r has been in a non-ready state for longer than 5 minutes.\nFiring 209:\n State: Resolved\n Started: 2026-10-13T13:07:44Z\n Ended: 2026-10-13T13:21:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-h8gwt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-h8gwt has been in a non-ready state for longer than 5 minutes.\nFiring 210:\n State: Resolved\n Started: 2026-10-13T13:07:45Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-pdbd9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-pdbd9 has been in a non-ready state for longer than 5 minutes.\nFiring 211:\n State: Resolved\n Started: 2026-10-13T13:07:45Z\n Ended: 2026-10-13T13:25:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-qhb62\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-qhb62 has been in a non-ready state for longer than 5 minutes.\nFiring 212:\n State: Resolved\n Started: 2026-10-13T13:07:45Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"ignition-server-proxy-96959d9bf-tnrhm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/ignition-server-proxy-96959d9bf-tnrhm has been in a non-ready state for longer than 5 minutes.\nFiring 213:\n State: Resolved\n Started: 2026-10-13T13:07:45Z\n Ended: 2026-10-13T13:25:31Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v\", pod=\"kube-controller-manager-7c6f78c4-99vg4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjq0klhk58c5rtaj9flc2hi84phla-p9a4v3n6g6e5b6v/kube-controller-manager-7c6f78c4-99vg4 has been in a non-ready state for longer than 5 minutes.\nFiring 214:\n State: Resolved\n Started: 2026-10-13T13:07:45Z\n Ended: 2026-10-13T13:25:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9120896-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v\", pod=\"ovnkube-control-plane-7db599cd56-9tn2z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tdtjv2mvp1nier7asur6or8ihbkq1og-d8q9r9d9f9e2k1v/ovnkube-control-plane-7db599cd56-9tn2z has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"e013dbbd062e97c554c577e1dca6673a390c8db815ec697cb882b13009dd878b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109977127079120896","occurred_at":"2026-10-13T11:58:43Z","failed_at":"alert","signature_id":"45a81790b69f5a13e41c3f6d58353b78d39c6df6f209d0a628107acd5871dc3b","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T13:33:58Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9120896-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.142:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9120896/providers/microsoft.monitor/accounts/services-j9120896\", namespace=\"klusterlet-2tdtjr3ne5vj3bi3oa6tbe1at5fpus2l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-hmgwl\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tdtjr3ne5vj3bi3oa6tbe1at5fpus2l on cluster ci01-j9120896-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","35f4b32d0c9a217f6e6dcfd9cb816a96a3bf1d26a1497dfd78708f5764de5a24","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734892052480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7484,"pr_state":"closed","pr_sha":"1857413cdb7ec426b141684f303bba8c60341534","final_merged_sha":"1857413cdb7ec426b141684f303bba8c60341534","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T11:41:15Z","started_at":"2026-10-13T11:41:15Z","completed_at":"2026-10-13T14:03:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T14:13:55.82536076Z","timing_metadata_checked_at":"2026-10-13T14:13:55.82536076Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T14:13:55.828978147Z","region_metadata_checked_at":"2026-10-13T14:13:55.828978147Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"0a9748bd1147dbee77461330a9544eb3d83b8bb28cc1d856690e5f009a98f5b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734892052480","occurred_at":"2026-10-13T11:41:15Z","failed_at":"alert","signature_id":"c0807ee7633cb3b74a1bec10eeb1a189d4d2920a14c2e55e84330c489057c130","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T13:37:23Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j2052480-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.129:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2052480/providers/microsoft.monitor/accounts/services-j2052480\", namespace=\"klusterlet-2tdtevmc4d8jve41e86ki97n3m7mpsvq\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-mlbkt\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tdtevmc4d8jve41e86ki97n3m7mpsvq on cluster ci01-j2052480-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"c62fef7e2da6ee8b26630da0ad27038efb0bb518f35b50df517a2a368928420d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7484/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734892052480","occurred_at":"2026-10-13T11:41:15Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T12:51:04Z\n Ended: 2026-10-13T13:11:03Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2052480-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734464233472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7485,"pr_state":"closed","pr_sha":"c6f94bff94d53889c16cfbe68c2fbf2f7e27f72c","final_merged_sha":"2c6f6382730d69d9ad5231a907568292ded88b2c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T11:41:15Z","started_at":"2026-10-13T11:41:15Z","completed_at":"2026-10-13T15:14:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T15:28:34.45364594Z","timing_metadata_checked_at":"2026-10-13T15:28:34.45364594Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T15:28:34.45654003Z","region_metadata_checked_at":"2026-10-13T15:28:34.45654003Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"20ddea912672f25268c0c1f39afb0c37f14d93ff22d3151b2827b0527da0c696","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734464233472","occurred_at":"2026-10-13T11:41:15Z","failed_at":"e2e","signature_id":"ed359cc4388f74eb3c414e5bd2ce69baff8941bc6e08595174f886722dcbacee","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x12402d3b2b10\u003e: \n resource group \"rg-candidate-4-23-qt7g4z697d5b\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-ch4n9f\" in resourcegroup=\"rg-candidate-4-23-qt7g4z697d5b\" to finish deleting: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\",\n \"name\": \"27faaa2f-8540-458b-a125-3f3536544cd1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T12:59:59.269440518Z\",\n \"endTime\": \"2026-10-13T13:24:08.339110012Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdtgm7u9ipde1epc2rt5pe43isp14m7 still exists (deletion dispatched at 2026-10-13T12:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n correlation ID: 365de814-fd9f-4b17-8525-a2096af1cada\n client request ID: 166f0483-5054-4255-b484-02e488295af5\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:302]: Unexpected error:\n \u003c*errors.joinError | 0x12402d3b2b10\u003e: \n resource group \"rg-candidate-4-23-qt7g4z697d5b\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-ch4n9f\" in resourcegroup=\"rg-candidate-4-23-qt7g4z697d5b\" to finish deleting: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\",\n \"name\": \"27faaa2f-8540-458b-a125-3f3536544cd1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T12:59:59.269440518Z\",\n \"endTime\": \"2026-10-13T13:24:08.339110012Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tdtgm7u9ipde1epc2rt5pe43isp14m7 still exists (deletion dispatched at 2026-10-13T12:59:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/27faaa2f-8540-458b-a125-3f3536544cd1\n correlation ID: 365de814-fd9f-4b17-8525-a2096af1cada\n client request ID: 166f0483-5054-4255-b484-02e488295af5\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7b00d884b9121fd268447d271afcfe83e409041219994587880e368f74202f32","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a40079a410eed617a35716b1a185506e111fed085a507d4754f2c9704c5ba84f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109972734464233472","occurred_at":"2026-10-13T11:41:15Z","failed_at":"e2e","signature_id":"c12ed8153f187823b5da8c9f2feb1648642f6798384985cc1969cf91f57dcead","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-w5zxw6n76pst/cluster-candidate-4-20-dnk6t4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x4d01c384420\u003e: \n failed to create HCP cluster cluster-candidate-4-20-dnk6t4: failed waiting for cluster=\"cluster-candidate-4-20-dnk6t4\" in resourcegroup=\"rg-candidate-4-20-w5zxw6n76pst\" to finish creating: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n \"name\": \"d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T12:39:39.471658263Z\",\n \"endTime\": \"2026-10-13T13:08:48.195755915Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n correlation ID: 6fd24ae1-3485-4cf6-8f00-56a00457a1d3\n client request ID: 52d0a7ef-5194-4884-9aba-b6475a9fa220\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-w5zxw6n76pst/cluster-candidate-4-20-dnk6t4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x4d01c384420\u003e: \n failed to create HCP cluster cluster-candidate-4-20-dnk6t4: failed waiting for cluster=\"cluster-candidate-4-20-dnk6t4\" in resourcegroup=\"rg-candidate-4-20-w5zxw6n76pst\" to finish creating: GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n \"name\": \"d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T12:39:39.471658263Z\",\n \"endTime\": \"2026-10-13T13:08:48.195755915Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4233472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8dbdcd4-fba9-45cb-9ad1-2c429fbb8dfb\n correlation ID: 6fd24ae1-3485-4cf6-8f00-56a00457a1d3\n client request ID: 52d0a7ef-5194-4884-9aba-b6475a9fa220\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de8a7215be9cbe7df7ff3c38f8eba50243fbc0204a90552d4bf737d8d09cb701","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingOnInfrastructureReady: Cluster infrastructure is still provisioning; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7b00d884b9121fd268447d271afcfe83e409041219994587880e368f74202f32","de8a7215be9cbe7df7ff3c38f8eba50243fbc0204a90552d4bf737d8d09cb701"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109971633794977792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7491,"pr_state":"closed","pr_sha":"3a3bdf67b23afe5bc70d16d1c2e6bf1de52ad199","final_merged_sha":"3a3bdf67b23afe5bc70d16d1c2e6bf1de52ad199","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T11:36:53Z","started_at":"2026-10-13T11:36:53Z","completed_at":"2026-10-13T13:51:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T14:03:36.153101459Z","timing_metadata_checked_at":"2026-10-13T14:03:36.153101459Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T14:03:36.156742248Z","region_metadata_checked_at":"2026-10-13T14:03:36.156742248Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"3464dec563c32ee82fe69616cc028f016fc8806bacfe737a156e60ce6d561099","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109971633794977792","occurred_at":"2026-10-13T11:36:53Z","failed_at":"alert","signature_id":"d06483cd9806ffb19df33df26852e61ee3baea8bf5cda6e46715e7ad95a9157e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T12:43:21Z\n Ended: 2026-10-13T12:58:20Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4977792-svc\", component=\"backend\", name=\"csstatedump\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-13T12:43:21Z\n Ended: 2026-10-13T12:58:20Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4977792-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"5926aba6d3dcf1f1366eac08eab1c012c1851950c4bc6200e60d26e26a375b62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109971633794977792","occurred_at":"2026-10-13T11:36:53Z","failed_at":"e2e","signature_id":"c2743ca915206fd985839d671b5618ccb4332bb960f8c549fdcf65d90ddee3d2","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-mnl9xxqhhnd5/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-mnl9xxqhhnd5/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-c5gl2m96t572/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\""}},{"row_id":"e5dff61e31adb19e0bc08d650029a850617f2d55e950cde23f458eaedc844451","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7491/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109971633794977792","occurred_at":"2026-10-13T11:36:53Z","failed_at":"alert","signature_id":"d802fbeb68fcabadbc0eb4bc2365b8c44020b5b0b251941d4bface05433d26f5","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T13:32:58Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4977792-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.113:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4977792/providers/microsoft.monitor/accounts/hcps-j4977792\", namespace=\"ocm-arohcpci01-2tdtau6376nk4eg778am4h7lka7gfbpr-b6r1t5h3h7g0b9b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-r5hzc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tdtau6376nk4eg778am4h7lka7gfbpr-b6r1t5h3h7g0b9b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-13T13:32:58Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4977792-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.113:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4977792/providers/microsoft.monitor/accounts/hcps-j4977792\", namespace=\"ocm-arohcpci01-2tdtau6376nk4eg778am4h7lka7gfbpr-b6r1t5h3h7g0b9b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-r5hzc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tdtau6376nk4eg778am4h7lka7gfbpr-b6r1t5h3h7g0b9b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109958187410001920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7488,"pr_state":"closed","pr_sha":"e343803f365d228cc3e0070ed414c588a26b5e9f","final_merged_sha":"e343803f365d228cc3e0070ed414c588a26b5e9f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T10:43:27Z","started_at":"2026-10-13T10:43:27Z","completed_at":"2026-10-13T13:03:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T13:10:41.779033192Z","timing_metadata_checked_at":"2026-10-13T13:10:41.779033192Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T13:10:41.78271538Z","region_metadata_checked_at":"2026-10-13T13:10:41.78271538Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9dcc34adce0bb8b7cdf3c0dcb4c837d5abf1b502a807e6a74f560181fadfbc1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109958187410001920","occurred_at":"2026-10-13T10:43:27Z","failed_at":"e2e","signature_id":"7322f74e045c9377f4c356e763fe2af1a5554b1f96d50357c44c7e7728ee62bb","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x2219287fa880\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-v5gf99nj48r6\" to finish creating: GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\",\n \"name\": \"2199d8bb-869a-41e2-acec-31983e058033\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T11:37:06.655702009Z\",\n \"endTime\": \"2026-10-13T12:06:15.607847324Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.18 is Partial (want Completed), started 8m45s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n correlation ID: 436fe8be-4edd-4c43-bff6-c5ba6dee20f5\n client request ID: 88121473-7bcb-4b6c-858a-daddb3fc95db\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0x2219287fa880\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-v5gf99nj48r6\" to finish creating: GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\",\n \"name\": \"2199d8bb-869a-41e2-acec-31983e058033\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T11:37:06.655702009Z\",\n \"endTime\": \"2026-10-13T12:06:15.607847324Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.18 is Partial (want Completed), started 8m45s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j0001920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2199d8bb-869a-41e2-acec-31983e058033\n correlation ID: 436fe8be-4edd-4c43-bff6-c5ba6dee20f5\n client request ID: 88121473-7bcb-4b6c-858a-daddb3fc95db\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109952299815145472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6944,"pr_state":"closed","pr_sha":"acc3c9dcb5525e30870acfc06f9c2589f32b6cba","final_merged_sha":"9f563b9577a791755ffccb7db3e92eaafb79811d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T10:20:03Z","started_at":"2026-10-13T10:20:03Z","completed_at":"2026-10-13T12:29:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T12:39:27.758225743Z","timing_metadata_checked_at":"2026-10-13T12:39:27.758225743Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T12:39:27.761471232Z","region_metadata_checked_at":"2026-10-13T12:39:27.761471232Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"52c4b16a2065237dd86d0b5962d53181952fa8963d5338aa110c05739886676f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109952299815145472","occurred_at":"2026-10-13T10:20:03Z","failed_at":"alert","signature_id":"153f2db7e7ffaab6160a3f35da43a89c7da37790c513bc78fc32bb4dd3b5f592","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-13T11:38:40Z\n Ended: 2026-10-13T11:44:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5145472-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tds5npj51044ccirlodrkqfmqvpa3kh-t7y6j3e0f4o4f5n\", pod=\"cluster-network-operator-78d46c7fbb-8nwxt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tds5npj51044ccirlodrkqfmqvpa3kh-t7y6j3e0f4o4f5n/cluster-network-operator-78d46c7fbb-8nwxt has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109943465260355584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7440,"pr_state":"closed","pr_sha":"fad2e28a7c1bd16cac804b58dec4470199a08d67","final_merged_sha":"09587f6f09c91b53ea705711a675d523def6cece","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T09:44:57Z","started_at":"2026-10-13T09:44:57Z","completed_at":"2026-10-13T10:26:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T10:45:16.707140895Z","timing_metadata_checked_at":"2026-10-13T10:45:16.707140895Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T10:45:16.710555483Z","region_metadata_checked_at":"2026-10-13T10:45:16.710555483Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d501c398cf77bd50c0852567246a2ec84ccfa09689a7ae8cc376b15728353f77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7440/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109943465260355584","occurred_at":"2026-10-13T09:44:57Z","failed_at":"provision","signature_id":"d517e51fc5b09c819903bacd247416bcbbe167939922787f3a2438a125b3fdc2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/infra (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-13T10:18:54.749Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 description=\"Step infra\\n Kind: ARM\\n Template: templates/mgmt-infra.bicep\\n Parameters: configurations/mgmt-infra.tmpl.bicepparam\"\ntime=2026-10-13T10:18:55.539Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2\ntime=2026-10-13T10:18:56.816Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 deployment=fa5bd4309de472f0310387105d9534c5f4fd97370ffb7825457798baaa1e4cb1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0355584-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ffa5bd4309de472f0310387105d9534c5f4fd97370ffb7825457798baaa1e4cb1\ntime=2026-10-13T10:20:27.529Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=infra stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0355584-mgmt-2/providers/Microsoft.Resources/deployments/fa5bd4309de472f0310387105d9534c5f4fd97370ffb7825457798baaa1e4cb1/operationStatuses/08584097201492563595\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0355584-mgmt-2/providers/Microsoft.Resources/deployments/ah-ci01-me-j0355584-2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"VaultAlreadyExists\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The vault name 'ah-ci01-me-j0355584-2' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link https://go.microsoft.com/fwlink/?linkid=2147740.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"255883e0e1738fd02f2daa41d490a5ba122bcfd657ae1579f6fbfb04d4dff13d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code VaultAlreadyExists; detail message the vault name '\u003cvault-name\u003e' is already in use. Vault names are globally unique so it is possible that the name is already taken. If you are sure that the vault name was not taken then it is possible that a vault with the same name was recently deleted but not purged after being placed in a recoverable state. If the vault is in a recoverable state then the vault will need to be purged before reusing the name. For more information about VaultAlreadyExists, soft delete and purging a vault follow this link \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["255883e0e1738fd02f2daa41d490a5ba122bcfd657ae1579f6fbfb04d4dff13d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7490/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109939416398041088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7490,"pr_state":"closed","pr_sha":"39a985a0ad564153de9e7b900b74f359a62b1f64","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T09:28:52Z","started_at":"2026-10-13T09:28:52Z","completed_at":"2026-10-13T11:43:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T11:58:07.110753363Z","timing_metadata_checked_at":"2026-10-13T11:58:07.110753363Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T11:58:07.113393254Z","region_metadata_checked_at":"2026-10-13T11:58:07.113393254Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109934998684438528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7383,"pr_state":"closed","pr_sha":"e3e1535d2b85418ee01d84e97f093ce7c4cc66ee","final_merged_sha":"8c3c716869a1faf8b6825153d663f1c139b5fbf8","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T09:11:18Z","started_at":"2026-10-13T09:11:18Z","completed_at":"2026-10-13T11:45:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T11:58:06.937666936Z","timing_metadata_checked_at":"2026-10-13T11:58:06.937666936Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T11:58:06.940946225Z","region_metadata_checked_at":"2026-10-13T11:58:06.940946225Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2c22c441c0b499f2166fcf432effe096d5b3e8e2b25e69ffb364c61e289f3236","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109934998684438528","occurred_at":"2026-10-13T09:11:18Z","failed_at":"e2e","signature_id":"e726ff68ce5087e7b46536a414b0c4ce318e72dc425aac96edae158603185df3","test_name":"SRE advances a control plane to an SRE-pinned version","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.005s.\ncluster \"sre-version-pin-z4tqzw\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.005s.\ncluster \"sre-version-pin-z4tqzw\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd","failure_pattern":"cluster \"\u003ccluster\u003e\" did not complete the SRE-pinned version 4.21.36","search_query":"cluster \"sre-version-pin-sjbwq2\" did not complete the SRE-pinned version 4.21.36"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109933827857059840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7389,"pr_state":"closed","pr_sha":"45fd914d96c1b908f204765007e479aac04b9b32","final_merged_sha":"45fd914d96c1b908f204765007e479aac04b9b32","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-13T09:06:39Z","started_at":"2026-10-13T09:06:39Z","completed_at":"2026-10-13T11:30:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T11:47:18.896204972Z","timing_metadata_checked_at":"2026-10-13T11:47:18.896204972Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T11:47:18.898933763Z","region_metadata_checked_at":"2026-10-13T11:47:18.898933763Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109933810454892544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"f9f9f546696bce226d7b2c56078c6afa008c2089","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T09:06:35Z","started_at":"2026-10-13T09:06:35Z","completed_at":"2026-10-13T11:31:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T11:47:18.758944013Z","timing_metadata_checked_at":"2026-10-13T11:47:18.758944013Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T11:47:18.768464182Z","region_metadata_checked_at":"2026-10-13T11:47:18.768464182Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a43b33e6bd60395601a854ac097449aa8b0de7f2d7ca5185cf4d161000c34bfb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109933810454892544","occurred_at":"2026-10-13T09:06:35Z","failed_at":"e2e","signature_id":"719b91552139342f4f6f074922a7af2b8c76f3cf3df0a249b92551325953e60c","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x121182295b80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-thbvsvb7b8fq\" to finish creating: GET https://rp.j4892544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\",\n \"name\": \"6c590bff-d1a2-4a4d-8670-8ed88c881da1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T10:21:53.436571535Z\",\n \"endTime\": \"2026-10-13T10:50:54.288659415Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-thbvsvb7b8fq--managed/providers/Microsoft.Compute/virtualMachines/m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-thbvsvb7b8fq--managed/providers/Microsoft.Compute/virtualMachines/m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4892544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\n correlation ID: a74e8836-2a27-4d03-aee8-222257d39c0c\n client request ID: 47ce1976-0b05-4962-8dcb-6f506b053324\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x121182295b80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-thbvsvb7b8fq\" to finish creating: GET https://rp.j4892544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\",\n \"name\": \"6c590bff-d1a2-4a4d-8670-8ed88c881da1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T10:21:53.436571535Z\",\n \"endTime\": \"2026-10-13T10:50:54.288659415Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-thbvsvb7b8fq--managed/providers/Microsoft.Compute/virtualMachines/m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-thbvsvb7b8fq--managed/providers/Microsoft.Compute/virtualMachines/m4q0a7n6v3e2h7g-np-1-ckxxw-dkp6n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4892544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c590bff-d1a2-4a4d-8670-8ed88c881da1\n correlation ID: a74e8836-2a27-4d03-aee8-222257d39c0c\n client request ID: 47ce1976-0b05-4962-8dcb-6f506b053324\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7489/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109922150965055488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7489,"pr_state":"closed","pr_sha":"0504e91fdd4bf81f1b97f30cf97fe76d47d75f6c","final_merged_sha":"a800194beb9b2fac67229770cb4864563f1e4d17","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-13T08:20:15Z","started_at":"2026-10-13T08:20:15Z","completed_at":"2026-10-13T10:45:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T10:55:21.128661165Z","timing_metadata_checked_at":"2026-10-13T10:55:21.128661165Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T10:55:21.131475956Z","region_metadata_checked_at":"2026-10-13T10:55:21.131475956Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109920002722238464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7488,"pr_state":"closed","pr_sha":"e343803f365d228cc3e0070ed414c588a26b5e9f","final_merged_sha":"e343803f365d228cc3e0070ed414c588a26b5e9f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-13T08:11:43Z","started_at":"2026-10-13T08:11:43Z","completed_at":"2026-10-13T10:38:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T10:55:20.794158332Z","timing_metadata_checked_at":"2026-10-13T10:55:20.794158332Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T10:55:20.797542121Z","region_metadata_checked_at":"2026-10-13T10:55:20.797542121Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"98327e8bb0ca04bf6e7edd5e36a3028f5f1f6eaa24f789a562f4691bf26ccf0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7488/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109920002722238464","occurred_at":"2026-10-13T08:11:43Z","failed_at":"alert","signature_id":"a19d81f743cded8d0f7583e6aed1c76e28428956863bd4963ceab29f6d4d2f68","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T10:15:17Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j2238464-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.6:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2238464/providers/microsoft.monitor/accounts/services-j2238464\", namespace=\"klusterlet-2tdqe5b5src4hkonr0ghmop9gvt133es\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-bfr2k\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tdqe5b5src4hkonr0ghmop9gvt133es on cluster ci01-j2238464-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109918681554227200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7457,"pr_state":"closed","pr_sha":"0c8ab0be284b6bd46607bbfca3c59a3ade65c50b","final_merged_sha":"0c8ab0be284b6bd46607bbfca3c59a3ade65c50b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-13T08:06:27Z","started_at":"2026-10-13T08:06:27Z","completed_at":"2026-10-13T10:29:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T10:45:16.727959826Z","timing_metadata_checked_at":"2026-10-13T10:45:16.727959826Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T10:45:16.730717417Z","region_metadata_checked_at":"2026-10-13T10:45:16.730717417Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109804957828911104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7485,"pr_state":"closed","pr_sha":"c6f94bff94d53889c16cfbe68c2fbf2f7e27f72c","final_merged_sha":"2c6f6382730d69d9ad5231a907568292ded88b2c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T00:34:34Z","started_at":"2026-10-13T00:34:34Z","completed_at":"2026-10-13T07:20:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T07:34:58.041557989Z","timing_metadata_checked_at":"2026-10-13T07:34:58.041557989Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T07:34:58.043961281Z","region_metadata_checked_at":"2026-10-13T07:34:58.043961281Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"59573290089a64d7661f990471ac7161490b7544c1d0ecac3b4f8298b7a8ba89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109804957828911104","occurred_at":"2026-10-13T00:34:34Z","failed_at":"e2e","signature_id":"241100486d1e2e5f6bf10dfd23c29984ebe1db934e3f4042fdf918032b8eac38","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:139]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xd6f1ba92440\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tk2gb5zvpbxs\" to finish creating: GET https://rp.j8911104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c1a3db7-a32b-4135-a38d-d0c3877758f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c1a3db7-a32b-4135-a38d-d0c3877758f3\",\n \"name\": \"6c1a3db7-a32b-4135-a38d-d0c3877758f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T01:35:58.975421113Z\",\n \"endTime\": \"2026-10-13T02:05:06.821915606Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-w6pg8-mtm5c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tk2gb5zvpbxs--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-w6pg8-mtm5c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-w6pg8-mtm5c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tk2gb5zvpbxs--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-w6pg8-mtm5c to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8911104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c1a3db7-a32b-4135-a38d-d0c3877758f3\n correlation ID: 3c00069f-3253-4f6a-bf00-774f9195172f\n client request ID: 439e929c-f0c1-47b9-b779-b67e3bd14fff\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:139]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xd6f1ba92440\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tk2gb5zvpbxs\" to finish creating: GET https://rp.j8911104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c1a3db7-a32b-4135-a38d-d0c3877758f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c1a3db7-a32b-4135-a38d-d0c3877758f3\",\n \"name\": \"6c1a3db7-a32b-4135-a38d-d0c3877758f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T01:35:58.975421113Z\",\n \"endTime\": \"2026-10-13T02:05:06.821915606Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-w6pg8-mtm5c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tk2gb5zvpbxs--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-w6pg8-mtm5c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-w6pg8-mtm5c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tk2gb5zvpbxs--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-w6pg8-mtm5c to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8911104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6c1a3db7-a32b-4135-a38d-d0c3877758f3\n correlation ID: 3c00069f-3253-4f6a-bf00-774f9195172f\n client request ID: 439e929c-f0c1-47b9-b779-b67e3bd14fff","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-dpn7kns9h5s6\" to finish creating: GET https://rp"}},{"row_id":"b60108c797dc2a18b8d9b45d4c1e57ea98c051948fd60521831e4c0200c55ccb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109804957828911104","occurred_at":"2026-10-13T00:34:34Z","failed_at":"alert","signature_id":"2782cf0dfc0a0c20b3659d17b6c3a4c2d2b8a0833ca169e4ccba3462287f86c1","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T02:25:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8911104-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.182:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8911104/providers/microsoft.monitor/accounts/hcps-j8911104\", namespace=\"ocm-arohcpci01-2tdjlphid04r183f4uvlk38j32sjcgf8-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-ctvv6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tdjlphid04r183f4uvlk38j32sjcgf8-v20260901/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-13T02:25:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8911104-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.182:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8911104/providers/microsoft.monitor/accounts/hcps-j8911104\", namespace=\"ocm-arohcpci01-2tdjlphid04r183f4uvlk38j32sjcgf8-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-ctvv6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tdjlphid04r183f4uvlk38j32sjcgf8-v20260901/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109803435254616064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7439,"pr_state":"closed","pr_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","final_merged_sha":"cb92dba9050dfae55af2d4a95c80e5682d65ce13","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-13T00:28:31Z","started_at":"2026-10-13T00:28:31Z","completed_at":"2026-10-13T07:19:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T07:34:57.781412591Z","timing_metadata_checked_at":"2026-10-13T07:34:57.781412591Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T07:34:57.784258882Z","region_metadata_checked_at":"2026-10-13T07:34:57.784258882Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109798012107624448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7162,"pr_state":"closed","pr_sha":"21c5097072b27775aeb5bfa1e803499de79c8013","final_merged_sha":"bf3311f63a37396e26e3b7417a174b8fca34b285","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T00:06:58Z","started_at":"2026-10-13T00:06:58Z","completed_at":"2026-10-13T07:21:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T07:34:57.39903477Z","timing_metadata_checked_at":"2026-10-13T07:34:57.39903477Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T07:34:57.401561462Z","region_metadata_checked_at":"2026-10-13T07:34:57.401561462Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"191459dc6e43c096c914ec38ff46edf7d310c6380413663bf6eabfe08a479477","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109798012107624448","occurred_at":"2026-10-13T00:06:58Z","failed_at":"e2e","signature_id":"17cac32c9f196891eb1cddda2b22294095ac725cb2ad07869f1b1d0d4e898ccd","test_name":"SRE advances a control plane to an SRE-pinned version","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.002s.\ncluster \"sre-version-pin-sjbwq2\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_version_pin.go:172]: Timed out after 2700.002s.\ncluster \"sre-version-pin-sjbwq2\" did not complete the SRE-pinned version 4.21.36\nExpected\n \u003cstring\u003e: 4.21.36 (Partial)\nto equal\n \u003cstring\u003e: 4.21.36","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd","failure_pattern":"cluster \"\u003ccluster\u003e\" did not complete the SRE-pinned version 4.21.36","search_query":"cluster \"sre-version-pin-sjbwq2\" did not complete the SRE-pinned version 4.21.36"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bb4de136ebd0024068fdae71f01ccc40217c2ff8bba4e521e07b515791de06dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109797931459547136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6920,"pr_state":"open","pr_sha":"f9f9f546696bce226d7b2c56078c6afa008c2089","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-13T00:06:39Z","started_at":"2026-10-13T00:06:39Z","completed_at":"2026-10-13T07:20:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-13T07:34:57.293312196Z","timing_metadata_checked_at":"2026-10-13T07:34:57.293312196Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-13T07:34:57.296766185Z","region_metadata_checked_at":"2026-10-13T07:34:57.296766185Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"78dea47ac1c34e9cc66f61113f6c5d6d639e542f7d21de88f2895e773f2a21ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109797931459547136","occurred_at":"2026-10-13T00:06:39Z","failed_at":"alert","signature_id":"4a20c7552b13dbe39fee7488dec78de85e3480b9fcb8c75ca75d30c3902dc95f","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-13T02:15:50Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9547136-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9547136/providers/microsoft.monitor/accounts/services-j9547136\", namespace=\"klusterlet-2tdj9mm19ano4hsp1ejpct9vbiqi8t8u\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-448ch\", prometheus=\"prometheus/prometheus\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tdj9mm19ano4hsp1ejpct9vbiqi8t8u on cluster ci01-j9547136-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired"}},{"row_id":"af28b4668111e0396ef4fe467156e9c3b5fa64bde778e8d8fc3461c640718c62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109797931459547136","occurred_at":"2026-10-13T00:06:39Z","failed_at":"e2e","signature_id":"ea8a7a2dfe743c69bf60d5e3342f070fda004e14eff8a761750c50c706798fa0","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x18ba92cf6f8\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x18ba9177500\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x18ba9176100\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x18ba91760e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:186]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x18ba92cf6f8\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x18ba9177500\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x18ba9176100\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x18ba91760e0\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-8jckw.apps.aro.cilium-cl.n1zs.j9547136.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"c4c207c7fc2de20c1ef729fb7c00f36a3a7a542b5713ff432884554bc8b73d6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2109797931459547136","occurred_at":"2026-10-13T00:06:39Z","failed_at":"e2e","signature_id":"43851cc86df3904c329cff3884669981de4ca71371d688f448f3b74e28255f14","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d502f6120\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-pkbxzb\" in resourcegroup=\"rg-np-version-upgrade-pkbxzb-8r9ncw6xxd9f\" to finish creating: GET https://rp.j9547136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\",\n \"name\": \"d75dd276-e1d3-41db-8b80-3fc3ed007e5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T01:17:18.97912251Z\",\n \"endTime\": \"2026-10-13T01:46:20.604608943Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9547136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\n correlation ID: 4e3d5b66-dbd0-45e8-931a-5c34e6be67a5\n client request ID: 33b8ec2f-ff38-476b-ac0b-f1b2c13ca519\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d502f6120\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-pkbxzb\" in resourcegroup=\"rg-np-version-upgrade-pkbxzb-8r9ncw6xxd9f\" to finish creating: GET https://rp.j9547136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\",\n \"name\": \"d75dd276-e1d3-41db-8b80-3fc3ed007e5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-13T01:17:18.97912251Z\",\n \"endTime\": \"2026-10-13T01:46:20.604608943Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9547136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d75dd276-e1d3-41db-8b80-3fc3ed007e5e\n correlation ID: 4e3d5b66-dbd0-45e8-931a-5c34e6be67a5\n client request ID: 33b8ec2f-ff38-476b-ac0b-f1b2c13ca519\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}