{"meta":{"start_date":"2026-10-21","end_date":"2026-10-21","start_at":"2026-10-21T00:00:00Z","end_at":"2026-10-22T00:00:00Z","timezone":"UTC","generated_at":"2026-10-30T03:22:35Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":83,"failed_runs":52,"raw_occurrences":92,"matched_occurrences":90,"runs_affected":50},"rows":[{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":13.25301204819277,"anchor_occurrences":11,"after_last_push_count":40,"trend_counts":[10,9,1,2,13,8,11],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":147,"prior_last_seen_at":"2026-10-18T21:09:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":11}],"full_error_samples":["alert fired 40 time(s) (17 unknown, 23 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-21T19:17:35Z\n  Ended:       2026-10-21T19:27:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjbvidupgp9nlik7eg10fsh59vogi9q-k9q7k5x7d5s9g6p\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjbvidupgp9nlik7eg10fsh59vogi9q-k9q7k5x7d5s9g6p/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-21T19:18:35Z\n  Ended:       2026-10-21T19:29:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjbvo8p5rl6fb29feg2u1dlsmppl5br-cilium-cl\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjbvo8p5rl6fb29feg2u1dlsmppl5br-cilium-cl/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-21T19:19:34Z\n  Ended:       2026-10-21T19:29:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjbvrsgfcv134idooo6qgdvr42t8n6n-sre-hcp-cluster\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjbvrsgfcv134idooo6qgdvr42t8n6n-sre-hcp-cluster/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-21T19:20:33Z\n  Ended:       2026-10-21T19:28:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjbvv9tadp00611k0lgs7t2avvomnlg-l4m3x6u1z1a0c9u\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjbvv9tadp00611k0lgs7t2avvomnlg-l4m3x6u1z1a0c9u/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-21T19:20:33Z\n  Ended:       2026-10-21T19:29:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjc0bg3vri3dhs1ag8ialdt811kjgsr-n4j7n9z1g6c5x9u\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjc0bg3vri3dhs1ag8ialdt811kjgsr-n4j7n9z1g6c5x9u/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-21T19:20:33Z\n  Ended:       2026-10-21T19:28:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjc1se0ft7nkghlt40iu0k36buchj61-t8n1u1u8g5n4s0q\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjc1se0ft7nkghlt40iu0k36buchj61-t8n1u1u8g5n4s0q/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-21T19:20:33Z\n  Ended:       2026-10-21T19:29:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjc0bbun58ddhmh76bq5nrgimbkrbfu-j5e9k6c2t1w8y4y\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjc0bbun58ddhmh76bq5nrgimbkrbfu-j5e9k6c2t1w8y4y/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-21T19:20:33Z\n  Ended:       2026-10-21T19:27:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjbvutvgtgr8umvqql88ajad4lgh83d-ea-cluster\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjbvutvgtgr8umvqql88ajad4lgh83d-ea-cluster/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-21T19:20:33Z\n  Ended:       2026-10-21T19:28:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjbvu2jghqqammopbihdm0dua266fi5-private-ingress\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjbvu2jghqqammopbihdm0dua266fi5-private-ingress/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-10-21T19:21:35Z\n  Ended:       2026-10-21T19:29:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjbvo5nqsrtu7s8gb82nob2v0i374vg-z7e6i1u0k4w3u9y\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjbvo5nqsrtu7s8gb82nob2v0i374vg-z7e6i1u0k4w3u9y/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-10-21T19:21:35Z\n  Ended:       2026-10-21T19:29:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjc0q7gfdcimk889lv3jap60dbckk4v-n9e5q5l8r9b9g6l\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjc0q7gfdcimk889lv3jap60dbckk4v-n9e5q5l8r9b9g6l/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-10-21T19:21:35Z\n  Ended:       2026-10-21T19:27:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjc0smbcqrb2gmp6er6bfn3u5iaj34t-agg-cluster\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjc0smbcqrb2gmp6er6bfn3u5iaj34t-agg-cluster/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-10-21T19:21:35Z\n  Ended:       2026-10-21T19:29:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjbvv6evjhccd20n1g3tbu8ipmll113-h5a2r1t7x9p4q1s\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjbvv6evjhccd20n1g3tbu8ipmll113-h5a2r1t7x9p4q1s/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-10-21T19:22:33Z\n  Ended:       2026-10-21T19:29:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjbvo6loe3k9ebp5ulerq9htpom697h-v20261001\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjbvo6loe3k9ebp5ulerq9htpom697h-v20261001/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-10-21T19:22:33Z\n  Ended:       2026-10-21T19:29:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjc00j4931i0aiu6lhdns95hm39tnvb-l7y7k5n2f0z7s4f\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjc00j4931i0aiu6lhdns95hm39tnvb-l7y7k5n2f0z7s4f/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-10-21T19:22:33Z\n  Ended:       2026-10-21T19:29:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjc0rts3ebhkjfde65hvjnl880lkqc0-c0h6l9s0y7a2e1w\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjc0rts3ebhkjfde65hvjnl880lkqc0-c0h6l9s0y7a2e1w/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-10-21T19:22:34Z\n  Ended:       2026-10-21T19:29:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1024128-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tjc01gjo3kpogvpf3p4k8245voidlnd-t1v4y7h5e1t6w4t\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tjc01gjo3kpogvpf3p4k8245voidlnd-t1v4y7h5e1t6w4t/etcd-0 has been in a non-ready state for longer than 5 minutes.","alert fired 23 time(s) (13 unknown, 10 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-2nmct\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-2nmct has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-vkxg4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-vkxg4 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-v7scv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-v7scv has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-dv9w2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-dv9w2 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-rn59z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-rn59z has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-r5rmw\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-r5rmw has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-k4vpn\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-k4vpn has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-v2422\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-v2422 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-xm94d\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-xm94d has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-xbkbv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-xbkbv has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-bzxx9\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-bzxx9 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-2szds\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-2szds has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-10-21T17:29:47Z\n  Ended:       2026-10-21T17:38:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0147200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t\", pod=\"azure-file-csi-driver-controller-67c8cd5c96-mvtgc\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tja5ssdu0crilsa2dh6uqu5d769g4h6-e8b3q7j1e4i8v5t/azure-file-csi-driver-controller-67c8cd5c96-mvtgc has been in a non-ready state for longer than 5 minutes.","alert fired 14 time(s) (7 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-21T14:32:51Z\n  Ended:       2026-10-21T14:43:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9083904-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tj7kqm1us0ib8nfkht65c5gkioeorko-k5a0f3u2b1h3i6d\", pod=\"azure-disk-csi-driver-controller-67d8f6fb68-blg8t\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tj7kqm1us0ib8nfkht65c5gkioeorko-k5a0f3u2b1h3i6d/azure-disk-csi-driver-controller-67d8f6fb68-blg8t has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-21T14:32:51Z\n  Ended:       2026-10-21T14:43:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9083904-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tj7kqm1us0ib8nfkht65c5gkioeorko-k5a0f3u2b1h3i6d\", pod=\"azure-file-csi-driver-controller-7ccfdf6644-64j9p\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tj7kqm1us0ib8nfkht65c5gkioeorko-k5a0f3u2b1h3i6d/azure-file-csi-driver-controller-7ccfdf6644-64j9p has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-21T15:30:44Z\n  Ended:       2026-10-21T15:40:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9083904-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tj8166hsuqubnhvhjdq3a2dlhjeqtpc-m0t1y4l2g3r0w4c\", pod=\"cluster-storage-operator-6587998f7c-8j9g8\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tj8166hsuqubnhvhjdq3a2dlhjeqtpc-m0t1y4l2g3r0w4c/cluster-storage-operator-6587998f7c-8j9g8 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-21T15:30:44Z\n  Ended:       2026-10-21T15:40:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9083904-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tj8166hsuqubnhvhjdq3a2dlhjeqtpc-m0t1y4l2g3r0w4c\", pod=\"cluster-network-operator-7ddcb6875d-x8rjm\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tj8166hsuqubnhvhjdq3a2dlhjeqtpc-m0t1y4l2g3r0w4c/cluster-network-operator-7ddcb6875d-x8rjm has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-21T15:30:44Z\n  Ended:       2026-10-21T15:40:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9083904-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tj8166hsuqubnhvhjdq3a2dlhjeqtpc-m0t1y4l2g3r0w4c\", pod=\"csi-snapshot-controller-operator-86969ccc8c-pcv4n\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tj8166hsuqubnhvhjdq3a2dlhjeqtpc-m0t1y4l2g3r0w4c/csi-snapshot-controller-operator-86969ccc8c-pcv4n has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-21T15:30:44Z\n  Ended:       2026-10-21T15:40:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9083904-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tj8166hsuqubnhvhjdq3a2dlhjeqtpc-m0t1y4l2g3r0w4c\", pod=\"dns-operator-7d6d66cd69-mmzzh\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tj8166hsuqubnhvhjdq3a2dlhjeqtpc-m0t1y4l2g3r0w4c/dns-operator-7d6d66cd69-mmzzh has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-21T15:30:44Z\n  Ended:       2026-10-21T15:40:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9083904-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tj8166hsuqubnhvhjdq3a2dlhjeqtpc-m0t1y4l2g3r0w4c\", pod=\"ingress-operator-6c94f6f674-6p4n9\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tj8166hsuqubnhvhjdq3a2dlhjeqtpc-m0t1y4l2g3r0w4c/ingress-operator-6c94f6f674-6p4n9 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112972661071024128","occurred_at":"2026-10-21T18:21:54Z","signature_id":"e022cccd594fa3d5234cad0346752847b07ba70bccea042d5ae52de678e37cce","pr_number":7641,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112941576740147200","occurred_at":"2026-10-21T16:18:23Z","signature_id":"d61b3736e46fdbf47ff70c9228e61ce7f4e4b697233e876da715a99a1aaa3abf","pr_number":7544,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"8c7f6d1edaa5710c4125bc73a609b4c3d429a08ade13e46c8e4281fafb2c76ff","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7655/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112878574095568896","occurred_at":"2026-10-21T12:08:02Z","signature_id":"1530acf4dc6b5473c3fa02aeb25361cad3410ef09497facd003bf2d38473d64e","pr_number":7655,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112870236943290368","occurred_at":"2026-10-21T11:34:40Z","signature_id":"8e613d7d081620c809b34a2325867e2d492450e0082a7d30d538dda181bad67b","pr_number":7658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112847311775207424","occurred_at":"2026-10-21T10:03:48Z","signature_id":"20da892cf554fd3be7861a8e1e840c8f217207bad0e3ad79be7fb8f48232558b","pr_number":7598,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112825723625410560","occurred_at":"2026-10-21T08:37:56Z","signature_id":"b6bf5604b0a42f47195498438faa6210896199e460bcbe387e2a05e1316c5574","pr_number":7463,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112784863382736896","occurred_at":"2026-10-21T05:55:39Z","signature_id":"62abd52c9be7a62c85412f6b642e997e70ac903bc5c169054098335fe9e1c0d4","pr_number":7073,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112764404264079360","occurred_at":"2026-10-21T04:34:21Z","signature_id":"2b564b8cde7f6c1b595883fabc95b45a94ecdf47efcb0b891e247b6ae0df4d04","pr_number":7133,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112756690729308160","occurred_at":"2026-10-21T04:03:42Z","signature_id":"f363da24ce71689db48e1d6a1a21503d120643077f60c365572df3ec78b8abae","pr_number":7629,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7633/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112713279871651840","occurred_at":"2026-10-21T01:11:12Z","signature_id":"1d009b135a5414c7770cc43bd398794b3ff3c3880d7713b90f3843222802ad4b","pr_number":7633,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":9,"runs_affected":9,"failed_runs":9,"run_impact_percent":10.843373493975903,"anchor_occurrences":9,"after_last_push_count":23,"trend_counts":[18,10,1,0,1,3,9],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":70,"prior_last_seen_at":"2026-10-17T21:48:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":9}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-npskt6qgktdr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fb9cc54d40\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-npskt6qgktdr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \"dev\"; let rgn = \"centralus\"; let cl = \"ci01-j4218880-mgmt-1\"; let ns = \"ocm-arohcpci01-2tjdecblfn866doucbvsck82v6djrsrv-p7h2c2c6w4m5q7f\"; let start = datetime(2026-10-21T20:47:23.6514068Z);\n    let scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\n    | where objectKind in (\"Pod\", \"Deployment\")\n    | summarize arg_max(timestamp, *) by objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project UID = uid, Node = tostring(object.spec.nodeName);\n    let desired = toscalar(inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project tolong(object.spec.replicas));\n    let current = toscalar(pods | count);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\n    | where container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | where tostring(log.pod_namespace) == ns\n    | extend UID = tostring(log.pod_uid)\n    | summarize arg_max(timestamp, *) by UID\n    | project UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    pods\n    | join kind=leftouter summaries on UID\n    | extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n    | project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-npskt6qgktdr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j4218880-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tjdecblfn866doucbvsck82v6djrsrv-p7h2c2c6w4m5q7f\\\"; let start = datetime(2026-10-21T20:47:23.6514068Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3fb9cc54d20\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j4218880-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tjdecblfn866doucbvsck82v6djrsrv-p7h2c2c6w4m5q7f\\\"; let start = datetime(2026-10-21T20:47:23.6514068Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x3fb9be4c6e0\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x3fb9cc54d00\u003e{\n                    msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j4218880-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tjdecblfn866doucbvsck82v6djrsrv-p7h2c2c6w4m5q7f\\\"; let start = datetime(2026-10-21T20:47:23.6514068Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x3fb9dd08330\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-npskt6qgktdr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fb9cc54d40\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-npskt6qgktdr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \"dev\"; let rgn = \"centralus\"; let cl = \"ci01-j4218880-mgmt-1\"; let ns = \"ocm-arohcpci01-2tjdecblfn866doucbvsck82v6djrsrv-p7h2c2c6w4m5q7f\"; let start = datetime(2026-10-21T20:47:23.6514068Z);\n    let scope = strcat(env, \"/\", rgn, \"/\", cl, \"/\", ns);\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\n    | where objectKind in (\"Pod\", \"Deployment\")\n    | summarize arg_max(timestamp, *) by objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project UID = uid, Node = tostring(object.spec.nodeName);\n    let desired = toscalar(inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project tolong(object.spec.replicas));\n    let current = toscalar(pods | count);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\n    | where container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | where tostring(log.pod_namespace) == ns\n    | extend UID = tostring(log.pod_uid)\n    | summarize arg_max(timestamp, *) by UID\n    | project UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    pods\n    | join kind=leftouter summaries on UID\n    | extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\n    | project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-npskt6qgktdr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j4218880-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tjdecblfn866doucbvsck82v6djrsrv-p7h2c2c6w4m5q7f\\\"; let start = datetime(2026-10-21T20:47:23.6514068Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3fb9cc54d20\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j4218880-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tjdecblfn866doucbvsck82v6djrsrv-p7h2c2c6w4m5q7f\\\"; let start = datetime(2026-10-21T20:47:23.6514068Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x3fb9be4c6e0\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x3fb9cc54d00\u003e{\n                    msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"centralus\\\"; let cl = \\\"ci01-j4218880-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tjdecblfn866doucbvsck82v6djrsrv-p7h2c2c6w4m5q7f\\\"; let start = datetime(2026-10-21T20:47:23.6514068Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x3fb9dd08330\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2fae15cf05c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-21T19:48:49.875795Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:48:49.875795Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2fae15cf05a0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:48:49.875795Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x2fae15532370\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x2fae15cf0580\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:48:49.875795Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x2fae16f952c0\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2fae15cf05c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-21T19:48:49.875795Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.002s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:48:49.875795Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x2fae15cf05a0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:48:49.875795Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x2fae15532370\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x2fae15cf0580\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-jdsxztvcznpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:48:49.875795Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x2fae16f952c0\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30fb1a9da3c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-21T19:20:56.4222237Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:20:56.4222237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x30fb1a9da3a0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:20:56.4222237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x30fb198a6050\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x30fb1a9da380\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:20:56.4222237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x30fb1b7ece70\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30fb1a9da3c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-21T19:20:56.4222237Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:20:56.4222237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x30fb1a9da3a0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:20:56.4222237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x30fb198a6050\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x30fb1a9da380\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-2x98xj2zxslb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-21T19:20:56.4222237Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x30fb1b7ece70\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112993261764218880","occurred_at":"2026-10-21T19:43:25Z","signature_id":"6cc78c72e72a534a8244b362dc855b1f918ce301797ebfe831e794fe3086e613","pr_number":7573,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112980138302902272","occurred_at":"2026-10-21T18:51:36Z","signature_id":"493db03cbcab063d02a6f4e8c26ca273383e025dea6b1416c50615fb4a10daab","pr_number":7646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112976501128302592","occurred_at":"2026-10-21T18:37:09Z","signature_id":"16d491b71901c0eaa23fa4e081ddbc5191a87b46fcce271652def29ac4c5dca2","pr_number":7650,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112945930595471360","occurred_at":"2026-10-21T16:35:41Z","signature_id":"5764829abf3f6f292d02dc1db28f4caba9532a5903d2b8948392ad9655d9d0d3","pr_number":7592,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112937980879441920","occurred_at":"2026-10-21T16:04:05Z","signature_id":"85c79c4dba4df28c9bc4a9cba69f3ad7b9fcf60ff3e996af1d403809b0d3f82b","pr_number":7646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7441/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112931863092793344","occurred_at":"2026-10-21T15:39:47Z","signature_id":"92c84ffe51f83bdb9ecd1ad9f1726dd6eabfcd6c3d1d7966f6dd357f5fa776a1","pr_number":7441,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112909828169928704","occurred_at":"2026-10-21T14:12:13Z","signature_id":"ddee255bded22ff1e4068f54f9e98b8bda8e231d0cb46bfaa881065b5c9a7a7c","pr_number":7641,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112847311775207424","occurred_at":"2026-10-21T10:03:48Z","signature_id":"76a175eef9ad29187c39d91c7412a972a211acb964bf87fcdc9f4b1365f51dd0","pr_number":7598,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7548/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112735125815955456","occurred_at":"2026-10-21T02:38:01Z","signature_id":"0750193e0940bf85207abcb241c6f593d6d579569734ab2829d711a3fa609041","pr_number":7548,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":9.63855421686747,"anchor_occurrences":8,"after_last_push_count":85,"trend_counts":[11,6,0,0,6,12,8],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":259,"prior_last_seen_at":"2026-10-16T16:35:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":8}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-21T21:50:37Z\n  Ended:       2026-10-21T22:00:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9052672-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-21T21:32:19Z\n  Ended:       2026-10-21T21:51:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0788736-svc\", component=\"backend\", name=\"csstatedump\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-21T21:32:19Z\n  Ended:       2026-10-21T21:51:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0788736-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-21T21:09:58Z\n  Ended:       2026-10-21T21:23:00Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4444160-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113009860609052672","occurred_at":"2026-10-21T20:49:43Z","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113003778620788736","occurred_at":"2026-10-21T20:25:33Z","signature_id":"d06483cd9806ffb19df33df26852e61ee3baea8bf5cda6e46715e7ad95a9157e","pr_number":7669,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112996419844444160","occurred_at":"2026-10-21T19:55:58Z","signature_id":"c162a5faff1d028c6c475696bbe945954abd7c58d8b8a99052f72cd3c91b0380","pr_number":7592,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112870236943290368","occurred_at":"2026-10-21T11:34:40Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112849728864522240","occurred_at":"2026-10-21T10:13:24Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112847311775207424","occurred_at":"2026-10-21T10:03:48Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7598,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112784863382736896","occurred_at":"2026-10-21T05:55:39Z","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","pr_number":7073,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112713315011530752","occurred_at":"2026-10-21T01:11:21Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7629,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.6144578313253013,"anchor_occurrences":3,"after_last_push_count":13,"trend_counts":[7,2,0,0,3,7,3],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":29,"prior_last_seen_at":"2026-10-16T09:12:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x208e48105340\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-qzrclp\" in resourcegroup=\"rg-np-noedge-qzrclp-s7mgb584cvbg\" to finish creating: GET https://rp.j8936960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/94d119b5-933d-4011-900a-944cda0c66bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/94d119b5-933d-4011-900a-944cda0c66bd\",\n      \"name\": \"94d119b5-933d-4011-900a-944cda0c66bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T00:23:29.776626313Z\",\n      \"endTime\": \"2026-10-22T00:42:32.225121177Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r0b1s8w2x6j2z6y-npnoedge-4-20-wks4l-pwh8n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-qzrclp-s7mgb584cvbg-np-ne-qzrclp--managed/providers/Microsoft.Compute/virtualMachines/r0b1s8w2x6j2z6y-npnoedge-4-20-wks4l-pwh8n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r0b1s8w2x6j2z6y-npnoedge-4-20-wks4l-pwh8n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-qzrclp-s7mgb584cvbg-np-ne-qzrclp--managed/providers/Microsoft.Compute/virtualMachines/r0b1s8w2x6j2z6y-npnoedge-4-20-wks4l-pwh8n to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8936960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/94d119b5-933d-4011-900a-944cda0c66bd\n            correlation ID: 9d04cab9-1988-4f3b-99fa-6267fc865f5b\n            client request ID: e2044c6b-9c9e-4046-9821-e1d653a15511\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.42\nUnexpected error:\n    \u003c*fmt.wrapError | 0x208e48105340\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-qzrclp\" in resourcegroup=\"rg-np-noedge-qzrclp-s7mgb584cvbg\" to finish creating: GET https://rp.j8936960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/94d119b5-933d-4011-900a-944cda0c66bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/94d119b5-933d-4011-900a-944cda0c66bd\",\n      \"name\": \"94d119b5-933d-4011-900a-944cda0c66bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-22T00:23:29.776626313Z\",\n      \"endTime\": \"2026-10-22T00:42:32.225121177Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r0b1s8w2x6j2z6y-npnoedge-4-20-wks4l-pwh8n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-qzrclp-s7mgb584cvbg-np-ne-qzrclp--managed/providers/Microsoft.Compute/virtualMachines/r0b1s8w2x6j2z6y-npnoedge-4-20-wks4l-pwh8n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r0b1s8w2x6j2z6y-npnoedge-4-20-wks4l-pwh8n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-qzrclp-s7mgb584cvbg-np-ne-qzrclp--managed/providers/Microsoft.Compute/virtualMachines/r0b1s8w2x6j2z6y-npnoedge-4-20-wks4l-pwh8n to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8936960.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/94d119b5-933d-4011-900a-944cda0c66bd\n            correlation ID: 9d04cab9-1988-4f3b-99fa-6267fc865f5b\n            client request ID: e2044c6b-9c9e-4046-9821-e1d653a15511\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15c75a6b8be0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-nctrfg\" in resourcegroup=\"rg-np-version-upgrade-nctrfg-45222jrbbpsn\" to finish creating: GET https://rp.j4218880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ae99b05-0591-45be-992e-01736d913464\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ae99b05-0591-45be-992e-01736d913464\",\n      \"name\": \"7ae99b05-0591-45be-992e-01736d913464\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:13:06.466186109Z\",\n      \"endTime\": \"2026-10-21T21:32:09.368172425Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x7u1u4c8y7v8s7l-npupgrade-4-20-2k82t-k6pq4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nctrfg-45222jrbbpsn-np-upgrade-nc-dd441e35/providers/Microsoft.Compute/virtualMachines/x7u1u4c8y7v8s7l-npupgrade-4-20-2k82t-k6pq4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x7u1u4c8y7v8s7l-npupgrade-4-20-2k82t-k6pq4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nctrfg-45222jrbbpsn-np-upgrade-nc-dd441e35/providers/Microsoft.Compute/virtualMachines/x7u1u4c8y7v8s7l-npupgrade-4-20-2k82t-k6pq4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4218880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ae99b05-0591-45be-992e-01736d913464\n            correlation ID: e2ddc6fc-2b3c-4d77-8c0d-3a130a41d2d3\n            client request ID: 5ad213ec-faaf-4144-a539-72b8c88d6c1f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15c75a6b8be0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-nctrfg\" in resourcegroup=\"rg-np-version-upgrade-nctrfg-45222jrbbpsn\" to finish creating: GET https://rp.j4218880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ae99b05-0591-45be-992e-01736d913464\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ae99b05-0591-45be-992e-01736d913464\",\n      \"name\": \"7ae99b05-0591-45be-992e-01736d913464\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:13:06.466186109Z\",\n      \"endTime\": \"2026-10-21T21:32:09.368172425Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x7u1u4c8y7v8s7l-npupgrade-4-20-2k82t-k6pq4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nctrfg-45222jrbbpsn-np-upgrade-nc-dd441e35/providers/Microsoft.Compute/virtualMachines/x7u1u4c8y7v8s7l-npupgrade-4-20-2k82t-k6pq4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x7u1u4c8y7v8s7l-npupgrade-4-20-2k82t-k6pq4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nctrfg-45222jrbbpsn-np-upgrade-nc-dd441e35/providers/Microsoft.Compute/virtualMachines/x7u1u4c8y7v8s7l-npupgrade-4-20-2k82t-k6pq4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4218880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ae99b05-0591-45be-992e-01736d913464\n            correlation ID: e2ddc6fc-2b3c-4d77-8c0d-3a130a41d2d3\n            client request ID: 5ad213ec-faaf-4144-a539-72b8c88d6c1f\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-5msk5q\" for cluster \"cluster-candidate-4-21-5msk5q\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2326984a6a0\u003e: \n    failed to create NodePool np-5msk5q: failed waiting for nodepool=\"np-5msk5q\" for cluster \"cluster-candidate-4-21-5msk5q\" in resourcegroup=\"rg-candidate-4-21-sv65px5xxhg7\" to finish creating: GET https://rp.j7966080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1106c4fd-dc58-4724-97cb-b35ad7ddc532\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1106c4fd-dc58-4724-97cb-b35ad7ddc532\",\n      \"name\": \"1106c4fd-dc58-4724-97cb-b35ad7ddc532\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T15:12:47.403558891Z\",\n      \"endTime\": \"2026-10-21T15:31:50.806949626Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e7l9m0o7f4b1b7k-np-5msk5q-6jh4f-4xx9j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-sv65px5xxhg7-candidate-5msk5q--managed/providers/Microsoft.Compute/virtualMachines/e7l9m0o7f4b1b7k-np-5msk5q-6jh4f-4xx9j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e7l9m0o7f4b1b7k-np-5msk5q-6jh4f-4xx9j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-sv65px5xxhg7-candidate-5msk5q--managed/providers/Microsoft.Compute/virtualMachines/e7l9m0o7f4b1b7k-np-5msk5q-6jh4f-4xx9j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7966080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1106c4fd-dc58-4724-97cb-b35ad7ddc532\n            correlation ID: e5915f53-e0ec-4f8c-9b0b-55f6298c1316\n            client request ID: 7cdad002-adc5-457c-ae3d-798682bcbe96\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-5msk5q\" for cluster \"cluster-candidate-4-21-5msk5q\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2326984a6a0\u003e: \n    failed to create NodePool np-5msk5q: failed waiting for nodepool=\"np-5msk5q\" for cluster \"cluster-candidate-4-21-5msk5q\" in resourcegroup=\"rg-candidate-4-21-sv65px5xxhg7\" to finish creating: GET https://rp.j7966080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1106c4fd-dc58-4724-97cb-b35ad7ddc532\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1106c4fd-dc58-4724-97cb-b35ad7ddc532\",\n      \"name\": \"1106c4fd-dc58-4724-97cb-b35ad7ddc532\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T15:12:47.403558891Z\",\n      \"endTime\": \"2026-10-21T15:31:50.806949626Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e7l9m0o7f4b1b7k-np-5msk5q-6jh4f-4xx9j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-sv65px5xxhg7-candidate-5msk5q--managed/providers/Microsoft.Compute/virtualMachines/e7l9m0o7f4b1b7k-np-5msk5q-6jh4f-4xx9j to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e7l9m0o7f4b1b7k-np-5msk5q-6jh4f-4xx9j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-sv65px5xxhg7-candidate-5msk5q--managed/providers/Microsoft.Compute/virtualMachines/e7l9m0o7f4b1b7k-np-5msk5q-6jh4f-4xx9j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7966080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1106c4fd-dc58-4724-97cb-b35ad7ddc532\n            correlation ID: e5915f53-e0ec-4f8c-9b0b-55f6298c1316\n            client request ID: 7cdad002-adc5-457c-ae3d-798682bcbe96\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113041621648936960","occurred_at":"2026-10-21T22:55:55Z","signature_id":"5aa59780f0fad8452f44485b590b7cc3e8347e5503dd52584557d2449d054972","pr_number":7646,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112993261764218880","occurred_at":"2026-10-21T19:43:25Z","signature_id":"a58080365ddd5c0dc81618b6b6ae7a0d7694c530b17cce32b11248a6e99b8739","pr_number":7573,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7421/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112907679297966080","occurred_at":"2026-10-21T14:03:41Z","signature_id":"f566cc9977fabdc655c9e26717eee9b0c07bd678d61de942ab4ffa56cc0bae87","pr_number":7421,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":3,"after_last_push_count":16,"trend_counts":[0,0,0,0,1,5,3],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":136,"prior_last_seen_at":"2026-10-13T15:48:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2dc0b06220f0\u003e: \n    resource group \"des-encrypt-xb5tc64jjbk5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-xb5tc64jjbk5\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7d6b2c93-7730-4824-8846-3824a20bfdec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7d6b2c93-7730-4824-8846-3824a20bfdec\",\n      \"name\": \"7d6b2c93-7730-4824-8846-3824a20bfdec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:45:35.149437794Z\",\n      \"endTime\": \"2026-10-21T15:09:42.233654784Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj7ubn23jup03cfdsjlt8o475dmidid still exists (deletion dispatched at 2026-10-21T14:45:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7d6b2c93-7730-4824-8846-3824a20bfdec\n            correlation ID: 60235ac6-f0dd-4c09-b328-c07af59f6052\n            client request ID: 76e45f1e-02ec-4d1b-9036-bebaa6509dd0\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2dc0b06220f0\u003e: \n    resource group \"des-encrypt-xb5tc64jjbk5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"des-encrypt\" in resourcegroup=\"des-encrypt-xb5tc64jjbk5\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7d6b2c93-7730-4824-8846-3824a20bfdec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7d6b2c93-7730-4824-8846-3824a20bfdec\",\n      \"name\": \"7d6b2c93-7730-4824-8846-3824a20bfdec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:45:35.149437794Z\",\n      \"endTime\": \"2026-10-21T15:09:42.233654784Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj7ubn23jup03cfdsjlt8o475dmidid still exists (deletion dispatched at 2026-10-21T14:45:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7d6b2c93-7730-4824-8846-3824a20bfdec\n            correlation ID: 60235ac6-f0dd-4c09-b328-c07af59f6052\n            client request ID: 76e45f1e-02ec-4d1b-9036-bebaa6509dd0\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3784d74e8120\u003e: \n    resource group \"rg-zstream-upgrade-4-21-vxd5jn5bhkxj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-5dpxf2\" in resourcegroup=\"rg-zstream-upgrade-4-21-vxd5jn5bhkxj\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc3c7e87-6636-4e88-bae2-c31e0e4df79d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc3c7e87-6636-4e88-bae2-c31e0e4df79d\",\n      \"name\": \"bc3c7e87-6636-4e88-bae2-c31e0e4df79d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:39:39.382659348Z\",\n      \"endTime\": \"2026-10-21T15:03:42.226397448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj7kh8478418ujofeaedu0kqd3iab5p still exists (deletion dispatched at 2026-10-21T14:39:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc3c7e87-6636-4e88-bae2-c31e0e4df79d\n            correlation ID: 91639a6f-eb4e-41bf-9deb-689b316310a7\n            client request ID: 7eff7a5b-5235-4d57-89e8-fc1d52195b20\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3784d74e8120\u003e: \n    resource group \"rg-zstream-upgrade-4-21-vxd5jn5bhkxj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-5dpxf2\" in resourcegroup=\"rg-zstream-upgrade-4-21-vxd5jn5bhkxj\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc3c7e87-6636-4e88-bae2-c31e0e4df79d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc3c7e87-6636-4e88-bae2-c31e0e4df79d\",\n      \"name\": \"bc3c7e87-6636-4e88-bae2-c31e0e4df79d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:39:39.382659348Z\",\n      \"endTime\": \"2026-10-21T15:03:42.226397448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj7kh8478418ujofeaedu0kqd3iab5p still exists (deletion dispatched at 2026-10-21T14:39:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bc3c7e87-6636-4e88-bae2-c31e0e4df79d\n            correlation ID: 91639a6f-eb4e-41bf-9deb-689b316310a7\n            client request ID: 7eff7a5b-5235-4d57-89e8-fc1d52195b20\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3d91346be150\u003e: \n    resource group \"rg-np-version-upgrade-wxkr7l-gs2n5hfrc7cj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-wxkr7l\" in resourcegroup=\"rg-np-version-upgrade-wxkr7l-gs2n5hfrc7cj\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f609346f-a626-48c2-b3f7-2432c0d4f1c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f609346f-a626-48c2-b3f7-2432c0d4f1c1\",\n      \"name\": \"f609346f-a626-48c2-b3f7-2432c0d4f1c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T15:16:30.030111335Z\",\n      \"endTime\": \"2026-10-21T15:40:31.183335705Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj8166hsuqubnhvhjdq3a2dlhjeqtpc still exists (deletion dispatched at 2026-10-21T15:16:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f609346f-a626-48c2-b3f7-2432c0d4f1c1\n            correlation ID: 7043ac86-0234-4e2b-aec8-d9e0460c5a5c\n            client request ID: 8cc99b04-9a95-4db0-933a-c9718f38aca5\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x3d91346be150\u003e: \n    resource group \"rg-np-version-upgrade-wxkr7l-gs2n5hfrc7cj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-wxkr7l\" in resourcegroup=\"rg-np-version-upgrade-wxkr7l-gs2n5hfrc7cj\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f609346f-a626-48c2-b3f7-2432c0d4f1c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f609346f-a626-48c2-b3f7-2432c0d4f1c1\",\n      \"name\": \"f609346f-a626-48c2-b3f7-2432c0d4f1c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T15:16:30.030111335Z\",\n      \"endTime\": \"2026-10-21T15:40:31.183335705Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj8166hsuqubnhvhjdq3a2dlhjeqtpc still exists (deletion dispatched at 2026-10-21T15:16:30Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f609346f-a626-48c2-b3f7-2432c0d4f1c1\n            correlation ID: 7043ac86-0234-4e2b-aec8-d9e0460c5a5c\n            client request ID: 8cc99b04-9a95-4db0-933a-c9718f38aca5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"8aacc7fd38055f6b4278d6d55251d1b71554c1136df7cea638aa119393ab4024","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"b49db5480351e20cd8e0a673bbd30414ee87afb334d58d9d645d59fea08d2b08","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"c492f738ef35c6af3113f30d3f5cf4e5693d3ac88c03b91bd4f6cde6527187e6","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"55f249c389c8381077274234c503c49f7f7b8734eaf356caba055061beb868f8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.4096385542168677,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3240dc3871e0\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-chb4gbwwsmzl\" to finish creating: GET https://rp.j0328704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2c2b5e0-cf5c-4ee8-837f-773d00c86a5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2c2b5e0-cf5c-4ee8-837f-773d00c86a5c\",\n      \"name\": \"d2c2b5e0-cf5c-4ee8-837f-773d00c86a5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T19:55:46.076391095Z\",\n      \"endTime\": \"2026-10-21T20:14:46.636602132Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine o5d8k9t4u7s8l8x-autoscale-az-qfbfg-r6wk5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-chb4gbwwsmzl--managed/providers/Microsoft.Compute/virtualMachines/o5d8k9t4u7s8l8x-autoscale-az-qfbfg-r6wk5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine o5d8k9t4u7s8l8x-autoscale-az-qfbfg-r6wk5: Failed: interfaces failed to create or update. err: failed to get existing resource np-autoscaling-chb4gbwwsmzl--managed/o5d8k9t4u7s8l8x-autoscale-az-qfbfg-r6wk5-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-chb4gbwwsmzl--managed/providers/Microsoft.Network/networkInterfaces/o5d8k9t4u7s8l8x-autoscale-az-qfbfg-r6wk5-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after additional tokens... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0328704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2c2b5e0-cf5c-4ee8-837f-773d00c86a5c\n            correlation ID: d1604de5-04bc-4cfd-a5ad-f2222992db63\n            client request ID: 22866604-4f96-48c8-8339-c93f8fcbb1b3\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3240dc3871e0\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-chb4gbwwsmzl\" to finish creating: GET https://rp.j0328704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2c2b5e0-cf5c-4ee8-837f-773d00c86a5c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2c2b5e0-cf5c-4ee8-837f-773d00c86a5c\",\n      \"name\": \"d2c2b5e0-cf5c-4ee8-837f-773d00c86a5c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T19:55:46.076391095Z\",\n      \"endTime\": \"2026-10-21T20:14:46.636602132Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine o5d8k9t4u7s8l8x-autoscale-az-qfbfg-r6wk5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-chb4gbwwsmzl--managed/providers/Microsoft.Compute/virtualMachines/o5d8k9t4u7s8l8x-autoscale-az-qfbfg-r6wk5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine o5d8k9t4u7s8l8x-autoscale-az-qfbfg-r6wk5: Failed: interfaces failed to create or update. err: failed to get existing resource np-autoscaling-chb4gbwwsmzl--managed/o5d8k9t4u7s8l8x-autoscale-az-qfbfg-r6wk5-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-chb4gbwwsmzl--managed/providers/Microsoft.Network/networkInterfaces/o5d8k9t4u7s8l8x-autoscale-az-qfbfg-r6wk5-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after additional tokens... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j0328704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d2c2b5e0-cf5c-4ee8-837f-773d00c86a5c\n            correlation ID: d1604de5-04bc-4cfd-a5ad-f2222992db63\n            client request ID: 22866604-4f96-48c8-8339-c93f8fcbb1b3\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xde2a318980\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pzz9klzh962n\" to finish creating: GET https://rp.j9261056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5af4fa15-307d-466b-b9c9-c0a6855da512\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5af4fa15-307d-466b-b9c9-c0a6855da512\",\n      \"name\": \"5af4fa15-307d-466b-b9c9-c0a6855da512\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:36:54.679484558Z\",\n      \"endTime\": \"2026-10-21T14:56:01.447308725Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllMachinesReady is False: 2 of 4 machines are not ready\\nMachine h1n9d3h2q0n6y9o-autoscale-az-p4nbn-q4z78: Failed: interfaces failed to create or update. err: failed to get existing resource np-autoscaling-pzz9klzh962n--managed/h1n9d3h2q0n6y9o-autoscale-az-p4nbn-q4z78-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-pzz9klzh962n--managed/providers/Microsoft.Network/networkInterfaces/h1n9d3h2q0n6y9o-autoscale-az-p4nbn-q4z78-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' seconds after additional tokens... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9261056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5af4fa15-307d-466b-b9c9-c0a6855da512\n            correlation ID: 873f9122-b54e-46d0-9f76-ed588ac965d9\n            client request ID: 1463f1a9-eece-4333-af81-213c95629aa9\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xde2a318980\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-pzz9klzh962n\" to finish creating: GET https://rp.j9261056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5af4fa15-307d-466b-b9c9-c0a6855da512\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5af4fa15-307d-466b-b9c9-c0a6855da512\",\n      \"name\": \"5af4fa15-307d-466b-b9c9-c0a6855da512\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:36:54.679484558Z\",\n      \"endTime\": \"2026-10-21T14:56:01.447308725Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllMachinesReady is False: 2 of 4 machines are not ready\\nMachine h1n9d3h2q0n6y9o-autoscale-az-p4nbn-q4z78: Failed: interfaces failed to create or update. err: failed to get existing resource np-autoscaling-pzz9klzh962n--managed/h1n9d3h2q0n6y9o-autoscale-az-p4nbn-q4z78-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-pzz9klzh962n--managed/providers/Microsoft.Network/networkInterfaces/h1n9d3h2q0n6y9o-autoscale-az-p4nbn-q4z78-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' seconds after additional tokens... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9261056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5af4fa15-307d-466b-b9c9-c0a6855da512\n            correlation ID: 873f9122-b54e-46d0-9f76-ed588ac965d9\n            client request ID: 1463f1a9-eece-4333-af81-213c95629aa9\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112981681840328704","occurred_at":"2026-10-21T18:57:44Z","signature_id":"c2ff8c56af4a23aa77680be0f1931216ebfe7a67443910bb2fc337a37223d53b","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112896971529261056","occurred_at":"2026-10-21T13:21:08Z","signature_id":"a0ac74e1e9cfb5ba15971ed4fbff7c4a6411a0121672ebd1f3dc4939b34661c4","pr_number":7582,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.4096385542168677,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,1,0,0,1,0,2],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-16T20:03:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13054070020\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2lvhqm85ncwq\" to finish creating: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/15626b81-4f40-4cd8-b91d-a63c8ef6d9cb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/15626b81-4f40-4cd8-b91d-a63c8ef6d9cb\",\n      \"name\": \"15626b81-4f40-4cd8-b91d-a63c8ef6d9cb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:23:04.095189902Z\",\n      \"endTime\": \"2026-10-21T14:42:12.03625287Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g2v1j2n5e4d4v3a-nodepool-128gib-x54k8-xjpcf: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine g2v1j2n5e4d4v3a-nodepool-128gib-x54k8-xjpcf: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/15626b81-4f40-4cd8-b91d-a63c8ef6d9cb\n            correlation ID: 317d5d8b-fd68-46aa-9ea3-280cce8619ef\n            client request ID: 9d6ff53f-a6f9-4a01-87a2-797532ad561c\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13054070020\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-2lvhqm85ncwq\" to finish creating: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/15626b81-4f40-4cd8-b91d-a63c8ef6d9cb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/15626b81-4f40-4cd8-b91d-a63c8ef6d9cb\",\n      \"name\": \"15626b81-4f40-4cd8-b91d-a63c8ef6d9cb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:23:04.095189902Z\",\n      \"endTime\": \"2026-10-21T14:42:12.03625287Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g2v1j2n5e4d4v3a-nodepool-128gib-x54k8-xjpcf: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine g2v1j2n5e4d4v3a-nodepool-128gib-x54k8-xjpcf: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/15626b81-4f40-4cd8-b91d-a63c8ef6d9cb\n            correlation ID: 317d5d8b-fd68-46aa-9ea3-280cce8619ef\n            client request ID: 9d6ff53f-a6f9-4a01-87a2-797532ad561c\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36f5972f0be0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-bc4dk5\" in resourcegroup=\"rg-np-version-upgrade-bc4dk5-7d4d8bxtxcwp\" to finish creating: GET https://rp.j9440128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d7299924-a73f-441d-881d-7c973d7b4248\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d7299924-a73f-441d-881d-7c973d7b4248\",\n      \"name\": \"d7299924-a73f-441d-881d-7c973d7b4248\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:06:44.0057756Z\",\n      \"endTime\": \"2026-10-21T14:25:46.108337887Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a5s6i4v4i7m1o8n-npupgrade-4-20-cctdj-kqldf: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a5s6i4v4i7m1o8n-npupgrade-4-20-cctdj-kqldf: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9440128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d7299924-a73f-441d-881d-7c973d7b4248\n            correlation ID: 335f4e13-f2a1-4c45-8bda-c2e1119ff7b6\n            client request ID: d412cd2d-c02a-4e92-80cf-e84a62b46887\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36f5972f0be0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-bc4dk5\" in resourcegroup=\"rg-np-version-upgrade-bc4dk5-7d4d8bxtxcwp\" to finish creating: GET https://rp.j9440128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d7299924-a73f-441d-881d-7c973d7b4248\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d7299924-a73f-441d-881d-7c973d7b4248\",\n      \"name\": \"d7299924-a73f-441d-881d-7c973d7b4248\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:06:44.0057756Z\",\n      \"endTime\": \"2026-10-21T14:25:46.108337887Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a5s6i4v4i7m1o8n-npupgrade-4-20-cctdj-kqldf: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a5s6i4v4i7m1o8n-npupgrade-4-20-cctdj-kqldf: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9440128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d7299924-a73f-441d-881d-7c973d7b4248\n            correlation ID: 335f4e13-f2a1-4c45-8bda-c2e1119ff7b6\n            client request ID: d412cd2d-c02a-4e92-80cf-e84a62b46887\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"0fc1b45a62000f9058ca481446a23c3c46f4ec39449ee2e47c245c5320e5b68d","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7608/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112885462539440128","occurred_at":"2026-10-21T12:35:16Z","signature_id":"0691ed656b103ad5ab4eed5f7076ff6f67eb130e61c14703f1ffcaf32e6c328d","pr_number":7608,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.4096385542168677,"anchor_occurrences":2,"after_last_push_count":14,"trend_counts":[5,1,0,0,2,6,2],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":28,"prior_last_seen_at":"2026-10-16T13:20:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c48d8a2f6e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-b9m4k92gvj4g\" to finish creating: GET https://rp.j5137024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34253dc7-331a-43c6-b614-64cab812db2d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34253dc7-331a-43c6-b614-64cab812db2d\",\n      \"name\": \"34253dc7-331a-43c6-b614-64cab812db2d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T17:06:32.563128324Z\",\n      \"endTime\": \"2026-10-21T17:25:32.638298339Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5137024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34253dc7-331a-43c6-b614-64cab812db2d\n            correlation ID: 730b525c-8264-4232-94a3-62d2afb7c9db\n            client request ID: f267fbb0-aa2f-4e04-8d7a-e731b332311e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c48d8a2f6e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-b9m4k92gvj4g\" to finish creating: GET https://rp.j5137024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34253dc7-331a-43c6-b614-64cab812db2d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34253dc7-331a-43c6-b614-64cab812db2d\",\n      \"name\": \"34253dc7-331a-43c6-b614-64cab812db2d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T17:06:32.563128324Z\",\n      \"endTime\": \"2026-10-21T17:25:32.638298339Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5137024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34253dc7-331a-43c6-b614-64cab812db2d\n            correlation ID: 730b525c-8264-4232-94a3-62d2afb7c9db\n            client request ID: f267fbb0-aa2f-4e04-8d7a-e731b332311e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x164574f57840\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fcpwj2cw9k8v\" to finish creating: GET https://rp.j1735936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/649ff5b3-4b10-49f2-be45-b62cf479b6c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/649ff5b3-4b10-49f2-be45-b62cf479b6c7\",\n      \"name\": \"649ff5b3-4b10-49f2-be45-b62cf479b6c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T17:16:17.057110724Z\",\n      \"endTime\": \"2026-10-21T17:35:26.070571768Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1735936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/649ff5b3-4b10-49f2-be45-b62cf479b6c7\n            correlation ID: 7a498bd5-b28c-425f-9407-7345c3d78071\n            client request ID: b5fdf720-d04b-4b3d-91c7-235c845f2533\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x164574f57840\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fcpwj2cw9k8v\" to finish creating: GET https://rp.j1735936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/649ff5b3-4b10-49f2-be45-b62cf479b6c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/649ff5b3-4b10-49f2-be45-b62cf479b6c7\",\n      \"name\": \"649ff5b3-4b10-49f2-be45-b62cf479b6c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T17:16:17.057110724Z\",\n      \"endTime\": \"2026-10-21T17:35:26.070571768Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1735936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/649ff5b3-4b10-49f2-be45-b62cf479b6c7\n            correlation ID: 7a498bd5-b28c-425f-9407-7345c3d78071\n            client request ID: b5fdf720-d04b-4b3d-91c7-235c845f2533\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112935310475137024","occurred_at":"2026-10-21T15:53:29Z","signature_id":"1b4c0c65d80775c6425b6a7bd29593b2817434ae8c4341022e39a2981e78295e","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112933937901735936","occurred_at":"2026-10-21T15:48:01Z","signature_id":"774bc4e2b70bd139db76d992e8674e71088afa3ce5452f89be3db6856e813abf","pr_number":7065,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3bc88c6c8876f1c2ea5668dd924adc7c81d6e0349b9850342e67442ed3fe914","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.4096385542168677,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-10-05"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-09T11:01:56Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x1258c35821b0\u003e: \n    resource group \"pullsecret-test-tqqf49s4vksc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-tqqf49s4vksc\" to finish deleting: GET https://rp.j8690432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/34d62dc1-1d60-4614-9fd4-7a691fd278e4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/34d62dc1-1d60-4614-9fd4-7a691fd278e4\",\n      \"name\": \"34d62dc1-1d60-4614-9fd4-7a691fd278e4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T18:09:45.740287452Z\",\n      \"endTime\": \"2026-10-21T18:33:49.851023477Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8690432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/34d62dc1-1d60-4614-9fd4-7a691fd278e4\n            correlation ID: 73ce79e7-e503-497f-984d-52615b1cd398\n            client request ID: 38899611-f32f-4f30-8353-0d70856187d8\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x1258c35821b0\u003e: \n    resource group \"pullsecret-test-tqqf49s4vksc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-tqqf49s4vksc\" to finish deleting: GET https://rp.j8690432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/34d62dc1-1d60-4614-9fd4-7a691fd278e4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/34d62dc1-1d60-4614-9fd4-7a691fd278e4\",\n      \"name\": \"34d62dc1-1d60-4614-9fd4-7a691fd278e4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T18:09:45.740287452Z\",\n      \"endTime\": \"2026-10-21T18:33:49.851023477Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8690432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/34d62dc1-1d60-4614-9fd4-7a691fd278e4\n            correlation ID: 73ce79e7-e503-497f-984d-52615b1cd398\n            client request ID: 38899611-f32f-4f30-8353-0d70856187d8\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2e81818a0078\u003e: \n    resource group \"patch-tags-2h6vxldjkpd2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-2h6vxldjkpd2\" to finish deleting: GET https://rp.j5137024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e23640fa-33a8-41b4-bf0c-88ace3164eea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e23640fa-33a8-41b4-bf0c-88ace3164eea\",\n      \"name\": \"e23640fa-33a8-41b4-bf0c-88ace3164eea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T17:06:29.350825983Z\",\n      \"endTime\": \"2026-10-21T17:30:32.428573897Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5137024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e23640fa-33a8-41b4-bf0c-88ace3164eea\n            correlation ID: de464733-8684-42c7-8fd5-e0982a079068\n            client request ID: 2dec17a5-5e85-44c4-af64-7517b9464dd3\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2e81818a0078\u003e: \n    resource group \"patch-tags-2h6vxldjkpd2\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-2h6vxldjkpd2\" to finish deleting: GET https://rp.j5137024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e23640fa-33a8-41b4-bf0c-88ace3164eea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e23640fa-33a8-41b4-bf0c-88ace3164eea\",\n      \"name\": \"e23640fa-33a8-41b4-bf0c-88ace3164eea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T17:06:29.350825983Z\",\n      \"endTime\": \"2026-10-21T17:30:32.428573897Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5137024.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e23640fa-33a8-41b4-bf0c-88ace3164eea\n            correlation ID: de464733-8684-42c7-8fd5-e0982a079068\n            client request ID: 2dec17a5-5e85-44c4-af64-7517b9464dd3\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7664/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112949864508690432","occurred_at":"2026-10-21T16:51:19Z","signature_id":"90233bb4bf77e06ec95888f47688227922873d7fb7326572043c3777dfd86d52","pr_number":7664,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112935310475137024","occurred_at":"2026-10-21T15:53:29Z","signature_id":"cd382d3947e3ee250c57690cc6125806c199a5a4f26f6c0dd4e4963a258a862c","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.4096385542168677,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":2}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-22T00:15:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0588800-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-dkzx8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"181b057e-60cb-49ad-af81-7495d4c18ee7\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-dkzx8 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-22T00:15:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-s8wts\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4f27896e-6819-4b13-8573-f17ef370d5ca\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-s8wts (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-22T00:18:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-s8wts\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4f27896e-6819-4b13-8573-f17ef370d5ca\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-s8wts (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-22T00:18:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0588800-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.150:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7f6f9c56c5-dkzx8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"181b057e-60cb-49ad-af81-7495d4c18ee7\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7f6f9c56c5-dkzx8 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T08:50:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9212928-svc\", component=\"kubernetes-infrastructure\", container=\"cilium-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.52:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9212928/providers/microsoft.monitor/accounts/services-j9212928\", namespace=\"kube-system\", pod=\"cilium-ksfwq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c90d39c5-332e-4d3e-8b00-995e03055f7c\"\n  Description: Pod kube-system/cilium-ksfwq (cilium-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T08:50:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9212928-svc\", component=\"kubernetes-infrastructure\", container=\"cilium-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.52:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9212928/providers/microsoft.monitor/accounts/services-j9212928\", namespace=\"kube-system\", pod=\"cilium-ksfwq\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c90d39c5-332e-4d3e-8b00-995e03055f7c\"\n  Description: Pod kube-system/cilium-ksfwq (cilium-agent) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113042472400588800","occurred_at":"2026-10-21T22:59:18Z","signature_id":"b279bcbc59c5b5a6422124d88fba5dfbfc8d8e2d489cc8d1cb59f8f6be1197be","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112809978489212928","occurred_at":"2026-10-21T07:35:27Z","signature_id":"38cffa08f4ff317466d02884bfa6390054d5e2f6fab2086828bb3b54e63fe2e8","pr_number":7592,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.4096385542168677,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,1,0,1,1,0,2],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":5,"prior_last_seen_at":"2026-10-18T21:21:36Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":2}],"full_error_samples":["time=2026-10-21T22:45:12.476Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: IstioUpgrade\\n  AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-21T22:45:13.969Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3213568-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-21T22:45:13.969Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3213568-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-21T22:45:14.834Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3213568-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-21T22:45:14.834Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3213568-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-21T22:45:14.834Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j3213568-svc\ntime=2026-10-21T23:00:17.748Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","time=2026-10-21T02:49:42.974Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: IstioUpgrade\\n  AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-21T02:49:44.531Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8353280-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-21T02:49:44.531Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8353280-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-21T02:49:45.285Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8353280-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-21T02:49:45.285Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j8353280-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-21T02:49:45.285Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j8353280-svc\ntime=2026-10-21T03:04:48.226Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7668/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113022983743213568","occurred_at":"2026-10-21T21:41:51Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7668,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112724660528353280","occurred_at":"2026-10-21T01:56:26Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7073,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[4,0,0,0,0,1,2],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":17,"prior_last_seen_at":"2026-10-15T16:38:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE starts a control plane z-stream upgrade with an SRE pin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x279359bea228\u003e: \n    resource group \"admin-api-version-pin-f5l942lsxnxx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-version-pin-7g5shb\" in resourcegroup=\"admin-api-version-pin-f5l942lsxnxx\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c9617226-f7ce-4606-a2f9-eb711857548d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c9617226-f7ce-4606-a2f9-eb711857548d\",\n      \"name\": \"c9617226-f7ce-4606-a2f9-eb711857548d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:39:39.950651125Z\",\n      \"endTime\": \"2026-10-21T15:03:42.224683778Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj7lluppmfrbdhhdr0g5hdh5ec68bse still exists (deletion dispatched at 2026-10-21T14:39:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c9617226-f7ce-4606-a2f9-eb711857548d\n            correlation ID: 48d5a3fe-e637-4aff-9f54-63cebfd278ec\n            client request ID: d94bc996-f0e4-48a3-ab58-bf75318409cc\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x279359bea228\u003e: \n    resource group \"admin-api-version-pin-f5l942lsxnxx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-version-pin-7g5shb\" in resourcegroup=\"admin-api-version-pin-f5l942lsxnxx\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c9617226-f7ce-4606-a2f9-eb711857548d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c9617226-f7ce-4606-a2f9-eb711857548d\",\n      \"name\": \"c9617226-f7ce-4606-a2f9-eb711857548d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:39:39.950651125Z\",\n      \"endTime\": \"2026-10-21T15:03:42.224683778Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj7lluppmfrbdhhdr0g5hdh5ec68bse still exists (deletion dispatched at 2026-10-21T14:39:40Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c9617226-f7ce-4606-a2f9-eb711857548d\n            correlation ID: 48d5a3fe-e637-4aff-9f54-63cebfd278ec\n            client request ID: d94bc996-f0e4-48a3-ab58-bf75318409cc\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2cef6ec040a8\u003e: \n    resource group \"admin-api-breakglass-9q9nnh4t7882\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9q9nnh4t7882\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8eaf5e6c-659a-4bc4-8c15-d54c813f8e68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8eaf5e6c-659a-4bc4-8c15-d54c813f8e68\",\n      \"name\": \"8eaf5e6c-659a-4bc4-8c15-d54c813f8e68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:29:52.564744569Z\",\n      \"endTime\": \"2026-10-21T14:53:53.054355548Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj7od9dtldg9atesqujpab56gdtktc3 still exists (deletion dispatched at 2026-10-21T14:29:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8eaf5e6c-659a-4bc4-8c15-d54c813f8e68\n            correlation ID: ad5bbe53-529d-4033-95da-f2bd383b8927\n            client request ID: 1402c96d-416f-418a-a03d-6b48908f76d1\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2cef6ec040a8\u003e: \n    resource group \"admin-api-breakglass-9q9nnh4t7882\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9q9nnh4t7882\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8eaf5e6c-659a-4bc4-8c15-d54c813f8e68\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8eaf5e6c-659a-4bc4-8c15-d54c813f8e68\",\n      \"name\": \"8eaf5e6c-659a-4bc4-8c15-d54c813f8e68\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:29:52.564744569Z\",\n      \"endTime\": \"2026-10-21T14:53:53.054355548Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj7od9dtldg9atesqujpab56gdtktc3 still exists (deletion dispatched at 2026-10-21T14:29:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8eaf5e6c-659a-4bc4-8c15-d54c813f8e68\n            correlation ID: ad5bbe53-529d-4033-95da-f2bd383b8927\n            client request ID: 1402c96d-416f-418a-a03d-6b48908f76d1\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"523a55929f07ab949cc61e075712cbece4e2ea2e4d62ebe9859b67e733460973","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"f51a71ff96ae344c9ff1e6f736a155db570b411a4fde8c6f3c2aa67e20e6651a","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"32b6cd590560c71daf2a070380398b0529fe23b7b49eae487b85f9203c09dd75","failure_pattern":"ERROR CODE: InvalidResource; detail message EtcdEncryptionKMSKeyAccessValidation: KeyVaultNetworkAccessDenied: The control plane cannot reach etcd encryption key \"etcd-data-kms-encryption-key\" in Key Vault \"cust-kv-\u003cid\u003e\": Key Vault responded with HTTP 403 (ForbiddenByConnection): Public network access is disabled and request is not from a trusted service nor via an approved private link. Clusters created with etcd.dataEncryption.customerManaged.kms.visibility \"Private\" reach the Key Vault from the virtual network that contains the cluster's VNet integration subnet. Ensure the Key Vault has an approved private endpoint, and that a privatelink.vaultcore.azure.net private DNS zone with a record for the Key Vault is linked to that virtual network.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidResource","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xda1b3d9a500\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rlbt6k884zhh\" to finish creating: GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90fc1494-ffd6-415a-ac40-0adc1fc596dc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90fc1494-ffd6-415a-ac40-0adc1fc596dc\",\n      \"name\": \"90fc1494-ffd6-415a-ac40-0adc1fc596dc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:21:31.114383608Z\",\n      \"endTime\": \"2026-10-21T21:37:30.726056138Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"EtcdEncryptionKMSKeyAccessValidation: KeyVaultNetworkAccessDenied: The control plane cannot reach etcd encryption key \\\"etcd-data-kms-encryption-key\\\" in Key Vault \\\"cust-kv-pa2z63bxwn3cs\\\": Key Vault responded with HTTP 403 (ForbiddenByConnection): Public network access is disabled and request is not from a trusted service nor via an approved private link. Clusters created with etcd.dataEncryption.customerManaged.kms.visibility \\\"Private\\\" reach the Key Vault from the virtual network that contains the cluster's VNet integration subnet. Ensure the Key Vault has an approved private endpoint, and that a privatelink.vaultcore.azure.net private DNS zone with a record for the Key Vault is linked to that virtual network.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90fc1494-ffd6-415a-ac40-0adc1fc596dc\n            correlation ID: df67305e-b659-4d48-9094-7c36da447124\n            client request ID: 406a6784-9511-4782-b611-e92af52ad4ab\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xda1b3d9a500\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rlbt6k884zhh\" to finish creating: GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90fc1494-ffd6-415a-ac40-0adc1fc596dc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90fc1494-ffd6-415a-ac40-0adc1fc596dc\",\n      \"name\": \"90fc1494-ffd6-415a-ac40-0adc1fc596dc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:21:31.114383608Z\",\n      \"endTime\": \"2026-10-21T21:37:30.726056138Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"EtcdEncryptionKMSKeyAccessValidation: KeyVaultNetworkAccessDenied: The control plane cannot reach etcd encryption key \\\"etcd-data-kms-encryption-key\\\" in Key Vault \\\"cust-kv-pa2z63bxwn3cs\\\": Key Vault responded with HTTP 403 (ForbiddenByConnection): Public network access is disabled and request is not from a trusted service nor via an approved private link. Clusters created with etcd.dataEncryption.customerManaged.kms.visibility \\\"Private\\\" reach the Key Vault from the virtual network that contains the cluster's VNet integration subnet. Ensure the Key Vault has an approved private endpoint, and that a privatelink.vaultcore.azure.net private DNS zone with a record for the Key Vault is linked to that virtual network.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90fc1494-ffd6-415a-ac40-0adc1fc596dc\n            correlation ID: df67305e-b659-4d48-9094-7c36da447124\n            client request ID: 406a6784-9511-4782-b611-e92af52ad4ab\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ff3aa19a440\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mjjsfb5sc9cl\" to finish creating: GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b471f6-0b3c-4f58-9820-fa7265add151\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b471f6-0b3c-4f58-9820-fa7265add151\",\n      \"name\": \"f0b471f6-0b3c-4f58-9820-fa7265add151\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:21:31.052814653Z\",\n      \"endTime\": \"2026-10-21T21:37:10.593291759Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"EtcdEncryptionKMSKeyAccessValidation: KeyVaultNetworkAccessDenied: The control plane cannot reach etcd encryption key \\\"etcd-data-kms-encryption-key\\\" in Key Vault \\\"cust-kv-om5carnxwh2f6\\\": Key Vault responded with HTTP 403 (ForbiddenByConnection): Public network access is disabled and request is not from a trusted service nor via an approved private link. Clusters created with etcd.dataEncryption.customerManaged.kms.visibility \\\"Private\\\" reach the Key Vault from the virtual network that contains the cluster's VNet integration subnet. Ensure the Key Vault has an approved private endpoint, and that a privatelink.vaultcore.azure.net private DNS zone with a record for the Key Vault is linked to that virtual network.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b471f6-0b3c-4f58-9820-fa7265add151\n            correlation ID: c7b4c1ab-73f1-409f-8dbc-b609459a629b\n            client request ID: 2e59ce2b-1874-419f-a049-3854aa659093\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ff3aa19a440\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mjjsfb5sc9cl\" to finish creating: GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b471f6-0b3c-4f58-9820-fa7265add151\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b471f6-0b3c-4f58-9820-fa7265add151\",\n      \"name\": \"f0b471f6-0b3c-4f58-9820-fa7265add151\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:21:31.052814653Z\",\n      \"endTime\": \"2026-10-21T21:37:10.593291759Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"EtcdEncryptionKMSKeyAccessValidation: KeyVaultNetworkAccessDenied: The control plane cannot reach etcd encryption key \\\"etcd-data-kms-encryption-key\\\" in Key Vault \\\"cust-kv-om5carnxwh2f6\\\": Key Vault responded with HTTP 403 (ForbiddenByConnection): Public network access is disabled and request is not from a trusted service nor via an approved private link. Clusters created with etcd.dataEncryption.customerManaged.kms.visibility \\\"Private\\\" reach the Key Vault from the virtual network that contains the cluster's VNet integration subnet. Ensure the Key Vault has an approved private endpoint, and that a privatelink.vaultcore.azure.net private DNS zone with a record for the Key Vault is linked to that virtual network.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b471f6-0b3c-4f58-9820-fa7265add151\n            correlation ID: c7b4c1ab-73f1-409f-8dbc-b609459a629b\n            client request ID: 2e59ce2b-1874-419f-a049-3854aa659093\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7394/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113003926449033216","occurred_at":"2026-10-21T20:26:04Z","signature_id":"996979826670df804720e4d0fb6a0c034724efdb1a30ef9c1b525a0a814b4e2c","pr_number":7394,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7394/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113003926449033216","occurred_at":"2026-10-21T20:26:04Z","signature_id":"eef1069e0b5033b9dca88452f8a36bc5ecbff35bf0aead57c8b3ca04a3db4d3a","pr_number":7394,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:509]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x36772c952c50\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:509]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x36772c952c50\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"91d040845479741f19caef6a91f68919c4832c7a9780a1875181b914f9bc8708","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":13,"trend_counts":[1,1,0,0,3,2,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":35,"prior_last_seen_at":"2026-10-16T07:17:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36cd28d2c720\u003e: \n    DNS for route host agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x36cd28c224c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36cd28d2c720\u003e: \n    DNS for route host agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x36cd28c224c0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-272d9.apps.aro.g1g2s2z9p9f8q5l.qgj6.j6469632.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112950731366469632","occurred_at":"2026-10-21T16:54:45Z","signature_id":"1fff3484bd2d120ca304bdd30a8a6003e09630446a7b5534a0ef105eec656070","pr_number":7582,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":14,"prior_last_seen_at":"2026-10-12T07:37:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ea8925d4400\u003e: \n    DNS for route host agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1ea891ce22e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1ea8925d4400\u003e: \n    DNS for route host agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    {\n        msg: \"DNS for route host agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1ea891ce22e0\u003e{\n            msg: \"DNS for agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-5pdz7.apps.aro.v20260901.wbc4.j8936960.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113041621648936960","occurred_at":"2026-10-21T22:55:55Z","signature_id":"df49086c8a11eeae5247ac0ba0fffb5010be6158b0bf81deed3833df22da8495","pr_number":7646,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[1,1,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-12"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-16T12:31:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:111]: failed to get cluster-api-azure managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc16479d48a0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n            correlation ID: 9f0dbad1-e718-4aa2-b236-36e96b6de447\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:111]: failed to get cluster-api-azure managed identity\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc16479d48a0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: 429\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"429\",\n        \"message\": \"Request is temporarily throttled because tenant XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 5 seconds.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-02-07/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n            correlation ID: 9f0dbad1-e718-4aa2-b236-36e96b6de447\n            client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7650/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112976501128302592","occurred_at":"2026-10-21T18:37:09Z","signature_id":"40c0e83750dac211950380bdfe32d45579a203c55213dbac0803589ceedd04e0","pr_number":7650,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"014ae109c7be412bf0a14ac3a53b4f4ccc9939291a93db20291f0672727b8246","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '3' s... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-5jgckd\" for cluster \"cluster-candidate-4-22-5jgckd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32059462a000\u003e: \n    failed to create NodePool np-5jgckd: failed waiting for nodepool=\"np-5jgckd\" for cluster \"cluster-candidate-4-22-5jgckd\" in resourcegroup=\"rg-candidate-4-22-rnz97cmn598x\" to finish creating: GET https://rp.j1024128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d50fab0-864e-4bf4-9092-8e2ed8d00995\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d50fab0-864e-4bf4-9092-8e2ed8d00995\",\n      \"name\": \"8d50fab0-864e-4bf4-9092-8e2ed8d00995\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T19:20:40.084231689Z\",\n      \"endTime\": \"2026-10-21T19:39:46.06586035Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u3w2j2u3b3n4m0t-np-5jgckd-4ppsc-48r8p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-rnz97cmn598x-candidate-5jgckd--managed/providers/Microsoft.Compute/virtualMachines/u3w2j2u3b3n4m0t-np-5jgckd-4ppsc-48r8p to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u3w2j2u3b3n4m0t-np-5jgckd-4ppsc-48r8p: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-rnz97cmn598x-candidate-5jgckd--managed/u3w2j2u3b3n4m0t-np-5jgckd-4ppsc-48r8p-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-rnz97cmn598x-candidate-5jgckd--managed/providers/Microsoft.Network/networkInterfaces/u3w2j2u3b3n4m0t-np-5jgckd-4ppsc-48r8p-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '3' s... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1024128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d50fab0-864e-4bf4-9092-8e2ed8d00995\n            correlation ID: a2a0560d-7e26-4028-86b5-5b49e70771b9\n            client request ID: c2e68888-288a-4197-9beb-bcebbb7b170a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:177]: failed to create node pool \"np-5jgckd\" for cluster \"cluster-candidate-4-22-5jgckd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32059462a000\u003e: \n    failed to create NodePool np-5jgckd: failed waiting for nodepool=\"np-5jgckd\" for cluster \"cluster-candidate-4-22-5jgckd\" in resourcegroup=\"rg-candidate-4-22-rnz97cmn598x\" to finish creating: GET https://rp.j1024128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d50fab0-864e-4bf4-9092-8e2ed8d00995\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d50fab0-864e-4bf4-9092-8e2ed8d00995\",\n      \"name\": \"8d50fab0-864e-4bf4-9092-8e2ed8d00995\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T19:20:40.084231689Z\",\n      \"endTime\": \"2026-10-21T19:39:46.06586035Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u3w2j2u3b3n4m0t-np-5jgckd-4ppsc-48r8p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-rnz97cmn598x-candidate-5jgckd--managed/providers/Microsoft.Compute/virtualMachines/u3w2j2u3b3n4m0t-np-5jgckd-4ppsc-48r8p to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u3w2j2u3b3n4m0t-np-5jgckd-4ppsc-48r8p: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-4-22-rnz97cmn598x-candidate-5jgckd--managed/u3w2j2u3b3n4m0t-np-5jgckd-4ppsc-48r8p-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-rnz97cmn598x-candidate-5jgckd--managed/providers/Microsoft.Network/networkInterfaces/u3w2j2u3b3n4m0t-np-5jgckd-4ppsc-48r8p-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'e92b9f76-b040-4cf6-a4dd-5c8bdc759e69' exceeded. Please try again after '3' s... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1024128.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8d50fab0-864e-4bf4-9092-8e2ed8d00995\n            correlation ID: a2a0560d-7e26-4028-86b5-5b49e70771b9\n            client request ID: c2e68888-288a-4197-9beb-bcebbb7b170a\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112972661071024128","occurred_at":"2026-10-21T18:21:54Z","signature_id":"8c11420e6d7f5771d4cc85727c7a3d3a7128d6a50d26ef29ad34e6ab5449f5de","pr_number":7641,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"380d8dbe7d3319b96b4fa9e4d08247affc130a0532da70e97653769374210873","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds a... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe9fe1d4cc20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-drk5zr95nhk4\" to finish creating: GET https://rp.j6828160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4b75858-d08d-4fff-b2a6-1eaa8bb64568\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4b75858-d08d-4fff-b2a6-1eaa8bb64568\",\n      \"name\": \"e4b75858-d08d-4fff-b2a6-1eaa8bb64568\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T19:03:42.372924526Z\",\n      \"endTime\": \"2026-10-21T19:22:50.94711189Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u2g8b3t8g2z9g0c-np-1-98sr2-8zrxf: Failed: availabilitysets failed to create or update. err: failed to get existing resource fips-enabled-drk5zr95nhk4--managed/2tjbpg3o68qrhrlgableu4g1r57ub3r2_u2g8b3t8g2z9g0c-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-drk5zr95nhk4--managed/providers/Microsoft.Compute/availabilitySets/2tjbpg3o68qrhrlgableu4g1r57ub3r2_u2g8b3t8g2z9g0c-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '1' seconds a... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6828160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4b75858-d08d-4fff-b2a6-1eaa8bb64568\n            correlation ID: 89bccd47-951a-48a6-a441-8b8045c21068\n            client request ID: 9c8a9209-4abc-4bd6-b4b0-28c6073afcb6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe9fe1d4cc20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-drk5zr95nhk4\" to finish creating: GET https://rp.j6828160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4b75858-d08d-4fff-b2a6-1eaa8bb64568\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4b75858-d08d-4fff-b2a6-1eaa8bb64568\",\n      \"name\": \"e4b75858-d08d-4fff-b2a6-1eaa8bb64568\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T19:03:42.372924526Z\",\n      \"endTime\": \"2026-10-21T19:22:50.94711189Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u2g8b3t8g2z9g0c-np-1-98sr2-8zrxf: Failed: availabilitysets failed to create or update. err: failed to get existing resource fips-enabled-drk5zr95nhk4--managed/2tjbpg3o68qrhrlgableu4g1r57ub3r2_u2g8b3t8g2z9g0c-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-drk5zr95nhk4--managed/providers/Microsoft.Compute/availabilitySets/2tjbpg3o68qrhrlgableu4g1r57ub3r2_u2g8b3t8g2z9g0c-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'cd39c606-1f6a-4062-a5b9-497cd04c39fc' exceeded. Please try again after '1' seconds a... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6828160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4b75858-d08d-4fff-b2a6-1eaa8bb64568\n            correlation ID: 89bccd47-951a-48a6-a441-8b8045c21068\n            client request ID: 9c8a9209-4abc-4bd6-b4b0-28c6073afcb6\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7428/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112962887956828160","occurred_at":"2026-10-21T17:42:55Z","signature_id":"e5c8579534baba19ff452192921916fc3e8a94d072d96cf4b31a5610d1104fa0","pr_number":7428,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"445bea11baaa5e6a0f158d83698b151143249148184d0e3c92735ff6d2548efa","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-15T22:11:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x139ab7392a40\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2xb7hfs7wzsp\" to finish creating: GET https://rp.j3007872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/89019ec8-1120-4c25-a583-ac3d503cf668\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/89019ec8-1120-4c25-a583-ac3d503cf668\",\n      \"name\": \"89019ec8-1120-4c25-a583-ac3d503cf668\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T23:26:25.049762723Z\",\n      \"endTime\": \"2026-10-21T23:45:34.818809633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s0b8e5p5e1a2t1q-np-1-bqzxs-c4d2f: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2xb7hfs7wzsp--managed/providers/Microsoft.Compute/virtualMachines/s0b8e5p5e1a2t1q-np-1-bqzxs-c4d2f to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s0b8e5p5e1a2t1q-np-1-bqzxs-c4d2f: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-labels-taints-2xb7hfs7wzsp--managed/s0b8e5p5e1a2t1q-np-1-bqzxs-c4d2f-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2xb7hfs7wzsp--managed/providers/Microsoft.Network/networkInterfaces/s0b8e5p5e1a2t1q-np-1-bqzxs-c4d2f-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '2' seconds after additional tokens are available.... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3007872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/89019ec8-1120-4c25-a583-ac3d503cf668\n            correlation ID: 2530e3f0-1ee5-450b-9d0e-3e90d3290756\n            client request ID: bcb2fe64-318b-44b4-82d6-9001a598a161\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x139ab7392a40\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2xb7hfs7wzsp\" to finish creating: GET https://rp.j3007872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/89019ec8-1120-4c25-a583-ac3d503cf668\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/89019ec8-1120-4c25-a583-ac3d503cf668\",\n      \"name\": \"89019ec8-1120-4c25-a583-ac3d503cf668\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T23:26:25.049762723Z\",\n      \"endTime\": \"2026-10-21T23:45:34.818809633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s0b8e5p5e1a2t1q-np-1-bqzxs-c4d2f: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2xb7hfs7wzsp--managed/providers/Microsoft.Compute/virtualMachines/s0b8e5p5e1a2t1q-np-1-bqzxs-c4d2f to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s0b8e5p5e1a2t1q-np-1-bqzxs-c4d2f: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-labels-taints-2xb7hfs7wzsp--managed/s0b8e5p5e1a2t1q-np-1-bqzxs-c4d2f-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2xb7hfs7wzsp--managed/providers/Microsoft.Network/networkInterfaces/s0b8e5p5e1a2t1q-np-1-bqzxs-c4d2f-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '2' seconds after additional tokens are available.... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3007872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/89019ec8-1120-4c25-a583-ac3d503cf668\n            correlation ID: 2530e3f0-1ee5-450b-9d0e-3e90d3290756\n            client request ID: bcb2fe64-318b-44b4-82d6-9001a598a161\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7675/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113033770973007872","occurred_at":"2026-10-21T22:24:43Z","signature_id":"210fb8c8d71af9f02f358fb9b5077f88f6297196d64a941ea628c7450674b869","pr_number":7675,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,1,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ca35c792040\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-cc4cdqjlzb2g\" to finish creating: GET https://rp.j5494784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d31c4ce-e10b-4cce-835c-391a0bb4ac85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d31c4ce-e10b-4cce-835c-391a0bb4ac85\",\n      \"name\": \"5d31c4ce-e10b-4cce-835c-391a0bb4ac85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T19:13:43.567943485Z\",\n      \"endTime\": \"2026-10-21T19:32:47.211157666Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1z4m5p8d9d3i3v-np-1-cbfq7-5dn9v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-cc4cdqjlzb2g--managed/providers/Microsoft.Compute/virtualMachines/x1z4m5p8d9d3i3v-np-1-cbfq7-5dn9v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x1z4m5p8d9d3i3v-np-1-cbfq7-5dn9v: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-cc4cdqjlzb2g--managed/x1z4m5p8d9d3i3v-np-1-cbfq7-5dn9v-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-cc4cdqjlzb2g--managed/providers/Microsoft.Network/networkInterfaces/x1z4m5p8d9d3i3v-np-1-cbfq7-5dn9v-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to http... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5494784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d31c4ce-e10b-4cce-835c-391a0bb4ac85\n            correlation ID: 6e04a42b-11b2-4a9c-addd-af84a9c30cb9\n            client request ID: dbfea4e9-55ac-41dd-b8ed-ecc52650efa5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ca35c792040\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-cc4cdqjlzb2g\" to finish creating: GET https://rp.j5494784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d31c4ce-e10b-4cce-835c-391a0bb4ac85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d31c4ce-e10b-4cce-835c-391a0bb4ac85\",\n      \"name\": \"5d31c4ce-e10b-4cce-835c-391a0bb4ac85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T19:13:43.567943485Z\",\n      \"endTime\": \"2026-10-21T19:32:47.211157666Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1z4m5p8d9d3i3v-np-1-cbfq7-5dn9v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-cc4cdqjlzb2g--managed/providers/Microsoft.Compute/virtualMachines/x1z4m5p8d9d3i3v-np-1-cbfq7-5dn9v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x1z4m5p8d9d3i3v-np-1-cbfq7-5dn9v: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-cc4cdqjlzb2g--managed/x1z4m5p8d9d3i3v-np-1-cbfq7-5dn9v-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-cc4cdqjlzb2g--managed/providers/Microsoft.Network/networkInterfaces/x1z4m5p8d9d3i3v-np-1-cbfq7-5dn9v-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '1' seconds after additional tokens are available. Refer to http... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5494784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5d31c4ce-e10b-4cce-835c-391a0bb4ac85\n            correlation ID: 6e04a42b-11b2-4a9c-addd-af84a9c30cb9\n            client request ID: dbfea4e9-55ac-41dd-b8ed-ecc52650efa5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7667/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112965263845494784","occurred_at":"2026-10-21T17:52:30Z","signature_id":"e45ce1fb828c33324438a2fd4efc301c6f14bba480655265258e1db07f95935d","pr_number":7667,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0c63db7c7a6fddc64d7430007c03eee77247a740f7f506b72ca4ce006b8c2fb4","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-tk5gq6\" for cluster \"cluster-hs-pre-tk5gq6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28a51cc17480\u003e: \n    failed to create NodePool np-tk5gq6: failed waiting for nodepool=\"np-tk5gq6\" for cluster \"cluster-hs-pre-tk5gq6\" in resourcegroup=\"rg-hs-pre--c25l8fnlxbfp\" to finish creating: GET https://rp.j2080256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f420461f-bece-47f6-898e-248327371b0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f420461f-bece-47f6-898e-248327371b0b\",\n      \"name\": \"f420461f-bece-47f6-898e-248327371b0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T06:16:15.392203282Z\",\n      \"endTime\": \"2026-10-21T06:35:22.529847577Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b1f8d3l9s7p6k7c-np-tk5gq6-8t786-lnwcz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--c25l8fnlxbfp-tk5gq6--managed/providers/Microsoft.Compute/virtualMachines/b1f8d3l9s7p6k7c-np-tk5gq6-8t786-lnwcz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine b1f8d3l9s7p6k7c-np-tk5gq6-8t786-lnwcz: Failed: interfaces failed to create or update. err: failed to get existing resource rg-hs-pre--c25l8fnlxbfp-tk5gq6--managed/b1f8d3l9s7p6k7c-np-tk5gq6-8t786-lnwcz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--c25l8fnlxbfp-tk5gq6--managed/providers/Microsoft.Network/networkInterfaces/b1f8d3l9s7p6k7c-np-tk5gq6-8t786-lnwcz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '1' seconds after additional tokens are... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2080256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f420461f-bece-47f6-898e-248327371b0b\n            correlation ID: babfafd3-932a-4646-be0e-e8e3feb4e4fd\n            client request ID: 4b55642a-8b13-4351-9120-9e71849c75ec\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-tk5gq6\" for cluster \"cluster-hs-pre-tk5gq6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28a51cc17480\u003e: \n    failed to create NodePool np-tk5gq6: failed waiting for nodepool=\"np-tk5gq6\" for cluster \"cluster-hs-pre-tk5gq6\" in resourcegroup=\"rg-hs-pre--c25l8fnlxbfp\" to finish creating: GET https://rp.j2080256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f420461f-bece-47f6-898e-248327371b0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f420461f-bece-47f6-898e-248327371b0b\",\n      \"name\": \"f420461f-bece-47f6-898e-248327371b0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T06:16:15.392203282Z\",\n      \"endTime\": \"2026-10-21T06:35:22.529847577Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b1f8d3l9s7p6k7c-np-tk5gq6-8t786-lnwcz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--c25l8fnlxbfp-tk5gq6--managed/providers/Microsoft.Compute/virtualMachines/b1f8d3l9s7p6k7c-np-tk5gq6-8t786-lnwcz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine b1f8d3l9s7p6k7c-np-tk5gq6-8t786-lnwcz: Failed: interfaces failed to create or update. err: failed to get existing resource rg-hs-pre--c25l8fnlxbfp-tk5gq6--managed/b1f8d3l9s7p6k7c-np-tk5gq6-8t786-lnwcz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--c25l8fnlxbfp-tk5gq6--managed/providers/Microsoft.Network/networkInterfaces/b1f8d3l9s7p6k7c-np-tk5gq6-8t786-lnwcz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '1' seconds after additional tokens are... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2080256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f420461f-bece-47f6-898e-248327371b0b\n            correlation ID: babfafd3-932a-4646-be0e-e8e3feb4e4fd\n            client request ID: 4b55642a-8b13-4351-9120-9e71849c75ec\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7097/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112771199242080256","occurred_at":"2026-10-21T05:01:18Z","signature_id":"9974d8f0dc1ea4d31cab7dd88ffa70c88064d637a5dcb65ced96a01e239341cf","pr_number":7097,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":10,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":19,"prior_last_seen_at":"2026-10-14T19:09:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-spmngrg5sbjv/cluster-candidate-4-22-rgcfjt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x306e6fa50b60\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-rgcfjt: failed waiting for cluster=\"cluster-candidate-4-22-rgcfjt\" in resourcegroup=\"rg-candidate-4-22-spmngrg5sbjv\" to finish creating: GET https://rp.j4079360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2fe0ff0e-edb6-4d7f-be94-4e97f7a4cb44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2fe0ff0e-edb6-4d7f-be94-4e97f7a4cb44\",\n      \"name\": \"2fe0ff0e-edb6-4d7f-be94-4e97f7a4cb44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T05:26:28.567903276Z\",\n      \"endTime\": \"2026-10-21T05:55:29.560038066Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.19 is Partial (want Completed), started 22m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4079360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2fe0ff0e-edb6-4d7f-be94-4e97f7a4cb44\n            correlation ID: eb19cda9-4b93-4643-b784-46181a5fe6fb\n            client request ID: b8394245-59f5-4d02-b42c-bba6984ea587\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:124]: HCP cluster rg-candidate-4-22-spmngrg5sbjv/cluster-candidate-4-22-rgcfjt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x306e6fa50b60\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-rgcfjt: failed waiting for cluster=\"cluster-candidate-4-22-rgcfjt\" in resourcegroup=\"rg-candidate-4-22-spmngrg5sbjv\" to finish creating: GET https://rp.j4079360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2fe0ff0e-edb6-4d7f-be94-4e97f7a4cb44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2fe0ff0e-edb6-4d7f-be94-4e97f7a4cb44\",\n      \"name\": \"2fe0ff0e-edb6-4d7f-be94-4e97f7a4cb44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T05:26:28.567903276Z\",\n      \"endTime\": \"2026-10-21T05:55:29.560038066Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.19 is Partial (want Completed), started 22m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j4079360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2fe0ff0e-edb6-4d7f-be94-4e97f7a4cb44\n            correlation ID: eb19cda9-4b93-4643-b784-46181a5fe6fb\n            client request ID: b8394245-59f5-4d02-b42c-bba6984ea587\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112764404264079360","occurred_at":"2026-10-21T04:34:21Z","signature_id":"f8553bcb9854fd482156c9385750db5e1681b88cc39d335030617f8fd30c4960","pr_number":7133,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,0,2,3,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":23,"prior_last_seen_at":"2026-10-06T05:43:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e8393e0bee0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-gbdj5r\" in resourcegroup=\"rg-np-version-upgrade-gbdj5r-ql9t2zm9w49h\" to finish creating: GET https://rp.j5207424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6d0eb9e1-4d4f-4249-8838-c2cc78760cf4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6d0eb9e1-4d4f-4249-8838-c2cc78760cf4\",\n      \"name\": \"6d0eb9e1-4d4f-4249-8838-c2cc78760cf4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T11:27:06.832183502Z\",\n      \"endTime\": \"2026-10-21T11:46:10.834576567Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z0c9h6g7p4h7x7v-npupgrade-4-20-9ds9t-fxlcn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z0c9h6g7p4h7x7v-npupgrade-4-20-9ds9t-fxlcn: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5207424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6d0eb9e1-4d4f-4249-8838-c2cc78760cf4\n            correlation ID: ede1199e-dd9c-4499-8177-d42c4436eb90\n            client request ID: d0c900cc-972d-4998-89ab-51274a7d36d4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.43\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e8393e0bee0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-gbdj5r\" in resourcegroup=\"rg-np-version-upgrade-gbdj5r-ql9t2zm9w49h\" to finish creating: GET https://rp.j5207424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6d0eb9e1-4d4f-4249-8838-c2cc78760cf4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6d0eb9e1-4d4f-4249-8838-c2cc78760cf4\",\n      \"name\": \"6d0eb9e1-4d4f-4249-8838-c2cc78760cf4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T11:27:06.832183502Z\",\n      \"endTime\": \"2026-10-21T11:46:10.834576567Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z0c9h6g7p4h7x7v-npupgrade-4-20-9ds9t-fxlcn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z0c9h6g7p4h7x7v-npupgrade-4-20-9ds9t-fxlcn: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5207424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/6d0eb9e1-4d4f-4249-8838-c2cc78760cf4\n            correlation ID: ede1199e-dd9c-4499-8177-d42c4436eb90\n            client request ID: d0c900cc-972d-4998-89ab-51274a7d36d4\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112847311775207424","occurred_at":"2026-10-21T10:03:48Z","signature_id":"ce604756517465eb8ac41960cd566aa39e386fbcaffac555d21484d2f8252e7a","pr_number":7598,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[4,1,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":9,"prior_last_seen_at":"2026-10-16T05:08:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2f37765b920\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-7xsbkjfw6rsz\" to finish creating: GET https://rp.j2736896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a4c6551c-33dc-4004-bb01-0a2aa772eecc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a4c6551c-33dc-4004-bb01-0a2aa772eecc\",\n      \"name\": \"a4c6551c-33dc-4004-bb01-0a2aa772eecc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T06:55:23.209858283Z\",\n      \"endTime\": \"2026-10-21T07:14:31.602256912Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0l3p9l4t2w4v4g-np-1-rqp7k-l24nh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-7xsbkjfw6rsz--managed/providers/Microsoft.Compute/virtualMachines/l0l3p9l4t2w4v4g-np-1-rqp7k-l24nh to exist\\nMachine l0l3p9l4t2w4v4g-np-1-rqp7k-mhsz6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-7xsbkjfw6rsz--managed/providers/Microsoft.Compute/virtualMachines/l0l3p9l4t2w4v4g-np-1-rqp7k-mhsz6 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0l3p9l4t2w4v4g-np-1-rqp7k-l24nh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-7xsbkjfw6rsz--managed/providers/Microsoft.Compute/virtualMachines/l0l3p9l4t2w4v4g-np-1-rqp7k-l24nh to exist\\nMachine l0l3p9l4t2w4v4g-np-1-rqp7k-mhsz6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-7xsbkjfw6rsz--managed/providers/Microsoft.Compute/virtualMachines/l0l3p9l4t2w4v4g-np-1-rqp7k-mhsz6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2736896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a4c6551c-33dc-4004-bb01-0a2aa772eecc\n            correlation ID: 6ad3d41d-5643-43b0-aa39-e8dd37fe508a\n            client request ID: 2b4396d2-9eda-4e0c-b6dc-3f33c00bc172\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2f37765b920\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-7xsbkjfw6rsz\" to finish creating: GET https://rp.j2736896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a4c6551c-33dc-4004-bb01-0a2aa772eecc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a4c6551c-33dc-4004-bb01-0a2aa772eecc\",\n      \"name\": \"a4c6551c-33dc-4004-bb01-0a2aa772eecc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T06:55:23.209858283Z\",\n      \"endTime\": \"2026-10-21T07:14:31.602256912Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0l3p9l4t2w4v4g-np-1-rqp7k-l24nh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-7xsbkjfw6rsz--managed/providers/Microsoft.Compute/virtualMachines/l0l3p9l4t2w4v4g-np-1-rqp7k-l24nh to exist\\nMachine l0l3p9l4t2w4v4g-np-1-rqp7k-mhsz6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-7xsbkjfw6rsz--managed/providers/Microsoft.Compute/virtualMachines/l0l3p9l4t2w4v4g-np-1-rqp7k-mhsz6 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0l3p9l4t2w4v4g-np-1-rqp7k-l24nh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-7xsbkjfw6rsz--managed/providers/Microsoft.Compute/virtualMachines/l0l3p9l4t2w4v4g-np-1-rqp7k-l24nh to exist\\nMachine l0l3p9l4t2w4v4g-np-1-rqp7k-mhsz6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-7xsbkjfw6rsz--managed/providers/Microsoft.Compute/virtualMachines/l0l3p9l4t2w4v4g-np-1-rqp7k-mhsz6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2736896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/a4c6551c-33dc-4004-bb01-0a2aa772eecc\n            correlation ID: 6ad3d41d-5643-43b0-aa39-e8dd37fe508a\n            client request ID: 2b4396d2-9eda-4e0c-b6dc-3f33c00bc172\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112784863382736896","occurred_at":"2026-10-21T05:55:39Z","signature_id":"7f7afb05c62784981f76f51a55b15e7309d7d70c3dc370ed867d55d7ced2607f","pr_number":7073,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b7faca8aeefdc29419e81028e85b19c0597b3aec44bae88a7309b486de862501","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-t5fkw-frfcv to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-t5fkw-frfcv to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:143]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7664fd40020\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-l6t4l44h7mwd\" to finish creating: GET https://rp.j9308160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74947d21-0259-46f2-b622-dc34ae7ef665\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74947d21-0259-46f2-b622-dc34ae7ef665\",\n      \"name\": \"74947d21-0259-46f2-b622-dc34ae7ef665\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T05:20:31.296701641Z\",\n      \"endTime\": \"2026-10-21T05:39:32.421681481Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine des-encrypt-des-np-t5fkw-frfcv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-l6t4l44h7mwd--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-t5fkw-frfcv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine des-encrypt-des-np-t5fkw-frfcv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-l6t4l44h7mwd--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-t5fkw-frfcv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9308160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74947d21-0259-46f2-b622-dc34ae7ef665\n            correlation ID: 2689c7c9-dcff-46e2-8f5f-9f51c04d066b\n            client request ID: 1c9831ba-b564-46d5-9462-612964a4e4e1\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:143]: failed to create nodepool des-np with DES\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7664fd40020\u003e: \n    failed to create NodePool des-np: failed waiting for nodepool=\"des-np\" for cluster \"des-encrypt\" in resourcegroup=\"des-encrypt-l6t4l44h7mwd\" to finish creating: GET https://rp.j9308160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74947d21-0259-46f2-b622-dc34ae7ef665\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74947d21-0259-46f2-b622-dc34ae7ef665\",\n      \"name\": \"74947d21-0259-46f2-b622-dc34ae7ef665\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T05:20:31.296701641Z\",\n      \"endTime\": \"2026-10-21T05:39:32.421681481Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine des-encrypt-des-np-t5fkw-frfcv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-l6t4l44h7mwd--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-t5fkw-frfcv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine des-encrypt-des-np-t5fkw-frfcv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/des-encrypt-l6t4l44h7mwd--managed/providers/Microsoft.Compute/virtualMachines/des-encrypt-des-np-t5fkw-frfcv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9308160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74947d21-0259-46f2-b622-dc34ae7ef665\n            correlation ID: 2689c7c9-dcff-46e2-8f5f-9f51c04d066b\n            client request ID: 1c9831ba-b564-46d5-9462-612964a4e4e1\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7629/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112756690729308160","occurred_at":"2026-10-21T04:03:42Z","signature_id":"f723346f702cdb65f17454b5d7148b4dc0cb4e7a5e15d79b927eef8621b4b019","pr_number":7629,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"55b8871066fd50b3c1d25edf1d72a2e26ea07377031b11e4744051d4627c5f06","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-\u003cid\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-\u003cid\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x12fed7d17db8\u003e: \n    VerifyNodesReady failed: there are not ready nodes: agg-cluster-agg-np-a-7m9bd-99rcc (status=False, reason=KubeletNotReady, message=[PLEG is not healthy: pleg has yet to be successful, container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, CSINode is not yet initialized], since=2026-10-21T06:37:27Z)\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-sskzwsj6mcw6\" to finish creating: GET https://rp.j2080256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5549a46b-1e5e-4888-bbe8-6d3e268cb178\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5549a46b-1e5e-4888-bbe8-6d3e268cb178\",\n      \"name\": \"5549a46b-1e5e-4888-bbe8-6d3e268cb178\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T06:17:49.830044932Z\",\n      \"endTime\": \"2026-10-21T06:36:52.904336002Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-7m9bd-99rcc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-sskzwsj6mcw6--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-7m9bd-99rcc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-7m9bd-99rcc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-sskzwsj6mcw6--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-7m9bd-99rcc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2080256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5549a46b-1e5e-4888-bbe8-6d3e268cb178\n            correlation ID: d7122e31-231d-46b1-9fc6-c124c9ad9057\n            client request ID: b05e45db-d8fe-4015-bf19-f42edb0a0ae5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x12fed7d17db8\u003e: \n    VerifyNodesReady failed: there are not ready nodes: agg-cluster-agg-np-a-7m9bd-99rcc (status=False, reason=KubeletNotReady, message=[PLEG is not healthy: pleg has yet to be successful, container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, CSINode is not yet initialized], since=2026-10-21T06:37:27Z)\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-sskzwsj6mcw6\" to finish creating: GET https://rp.j2080256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5549a46b-1e5e-4888-bbe8-6d3e268cb178\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5549a46b-1e5e-4888-bbe8-6d3e268cb178\",\n      \"name\": \"5549a46b-1e5e-4888-bbe8-6d3e268cb178\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T06:17:49.830044932Z\",\n      \"endTime\": \"2026-10-21T06:36:52.904336002Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-7m9bd-99rcc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-sskzwsj6mcw6--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-7m9bd-99rcc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-7m9bd-99rcc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-sskzwsj6mcw6--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-7m9bd-99rcc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j2080256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5549a46b-1e5e-4888-bbe8-6d3e268cb178\n            correlation ID: d7122e31-231d-46b1-9fc6-c124c9ad9057\n            client request ID: b05e45db-d8fe-4015-bf19-f42edb0a0ae5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7097/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112771199242080256","occurred_at":"2026-10-21T05:01:18Z","signature_id":"4db7db485a4481dc862bb5937c848321e5d26a52d54b7be5e2484d8fdfd58ab4","pr_number":7097,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ce4a245abbe72e80695d293314fde109d7f8f029d86037d3ac7a222a08fa46a0","failure_pattern":"ERROR CODE: DeploymentFailed; detail code NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/logs-mdsd' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-21T16:44:29.860Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-10-21T16:44:30.303Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-10-21T16:44:30.918Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=64152644a53dcaa071ce9ff6422266013cd78714911c11a5dae06a36893119aa portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6080768-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F64152644a53dcaa071ce9ff6422266013cd78714911c11a5dae06a36893119aa\ntime=2026-10-21T16:46:31.557Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6080768-mgmt-2/providers/Microsoft.Resources/deployments/64152644a53dcaa071ce9ff6422266013cd78714911c11a5dae06a36893119aa/operationStatuses/08584090058149128475\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6080768-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-post\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6080768-mgmt-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/logs-mdsd' was not found.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"NotFound\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6080768-mgmt-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/logs-mdsd' was not found.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7394/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112939820866080768","occurred_at":"2026-10-21T16:11:24Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":7394,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":19,"prior_last_seen_at":"2026-09-29T09:10:58Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-21T18:14:19.941Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-10-21T18:27:57.001Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-21T18:14:19.952926036Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6794752-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6794752-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-10-21T18:14:22.491703162Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6794752-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6794752-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-21T18:14:22.594394416Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6794752-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6794752-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-10-21T18:27:56.998402918Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6794752-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6794752-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"8102785885B170C8\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6794752-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j6794752-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-21T18:23:06.0910187Z\\\",\\\"duration\\\":\\\"PT2M24.1616872S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-21T18:27:56.998464939Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6794752-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6794752-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"0BDA719BF64D0BED\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6794752-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j6794752-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-21T18:14:24.88832Z\\\",\\\"duration\\\":\\\"PT13M8.3947089S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-21T18:27:56.99847726Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6794752-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6794752-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E62198E06C157714\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6794752-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j6794752-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-21T18:23:06.0910187Z\\\",\\\"duration\\\":\\\"PT1M35.4239028S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n  - Availability Zone\\\\n  - Differencing (Ephemeral) Disks\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n  - VM Size\\\\n\\\",\\\"target\\\":\\\"2\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft1-13745880-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 3, 1, 2), 'Standard_D16ds_v6' (zones 2, 1, 3), 'Standard_D16ds_v6' (eastus2, zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft1-13745880-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n  - Availability Zone\\\\n  - Differencing (Ephemeral) Disks\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n  - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-21T18:27:56.998529601Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6794752-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6794752-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DE9772B750D369C1\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6794752-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j6794752-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-21T18:23:06.0910187Z\\\",\\\"duration\\\":\\\"PT1M36.8052474S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-21T18:27:56.998540041Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6794752-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6794752-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"C04DDBEF7CE6AEB7\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6794752-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j6794752-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-21T18:23:06.0910187Z\\\",\\\"duration\\\":\\\"PT2M53.1939471S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-21T18:27:56.998550371Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6794752-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6794752-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"7938AC3EB9262313\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6794752-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j6794752-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-21T18:14:24.88832Z\\\",\\\"duration\\\":\\\"PT8M41.1379305S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-21T18:27:56.998572712Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6794752-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6794752-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6794752-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584090004226967454\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n    \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": [\\\\n      {\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j6794752-mgmt-2/agentPools/userswft1\\\\\\\",\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n        \\\\\\\"details\\\\\\\": [\\\\n          {\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft1-13745880-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 3, 1, 2), 'Standard_D16ds_v6' (zones 2, 1, 3), 'Standard_D16ds_v6' (eastus2, zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6794752-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft1-13745880-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\",\\\\n            \\\\\\\"details\\\\\\\": [\\\\n              {\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"2\\\\\\\",\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n  - Availability Zone\\\\\\\\n  - Differencing (Ephemeral) Disks\\\\\\\\n  - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n  - VM Size\\\\\\\\n\\\\\\\"\\\\n              }\\\\n            ]\\\\n          }\\\\n        ]\\\\n      }\\\\n    ]\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"13m37.045666056s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112962903886794752","occurred_at":"2026-10-21T17:43:07Z","signature_id":"094523af15fa867d1934e797f870b70c2a4a82cbbdf4ed804bc7f849a7e43e75","pr_number":7178,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ce15c7f9d7dc2228f7cb53452c85e04ce353e45a3f1f6cb5dfe6faa878ab1521","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: externalAuths, serviceProviderClusters, serviceProviderExternalAuths; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0xd2af1258138\u003e: \n    resource group \"rg-candidate-4-20-hth2d5sgmqkq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-82k925\" in resourcegroup=\"rg-candidate-4-20-hth2d5sgmqkq\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff5d427f-8d05-44df-b7ea-04a0f7376b37\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff5d427f-8d05-44df-b7ea-04a0f7376b37\",\n      \"name\": \"ff5d427f-8d05-44df-b7ea-04a0f7376b37\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:40:14.70614785Z\",\n      \"endTime\": \"2026-10-21T15:04:22.445579694Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj7kg5i5v3qeqbhhu1aj4t26p20km4l still exists (deletion dispatched at 2026-10-21T14:40:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/externalauths/serviceProviderExternalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff5d427f-8d05-44df-b7ea-04a0f7376b37\n            correlation ID: fc2328c7-eacc-4077-91a9-9b41a9c39453\n            client request ID: 7963c683-9f71-48b3-b976-d344efc9d8b4\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0xd2af1258138\u003e: \n    resource group \"rg-candidate-4-20-hth2d5sgmqkq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-82k925\" in resourcegroup=\"rg-candidate-4-20-hth2d5sgmqkq\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff5d427f-8d05-44df-b7ea-04a0f7376b37\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff5d427f-8d05-44df-b7ea-04a0f7376b37\",\n      \"name\": \"ff5d427f-8d05-44df-b7ea-04a0f7376b37\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:40:14.70614785Z\",\n      \"endTime\": \"2026-10-21T15:04:22.445579694Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj7kg5i5v3qeqbhhu1aj4t26p20km4l still exists (deletion dispatched at 2026-10-21T14:40:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/externalauths/serviceProviderExternalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ff5d427f-8d05-44df-b7ea-04a0f7376b37\n            correlation ID: fc2328c7-eacc-4077-91a9-9b41a9c39453\n            client request ID: 7963c683-9f71-48b3-b976-d344efc9d8b4\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"ce0adfa3c23ea1bdaf827e684a097500a0c190da5c3dfcdeb28633dabaf391b6","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"82fcc2cca51beebf00c7302e8f8dc3563b7995a4550dbcf53d6d84d1820ed342","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-08T11:12:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0xfdd760f41e0\u003e: \n    resource group \"rg-np-skip-minor-fgcrg6-74ql4tphgbdm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-fgcrg6\" in resourcegroup=\"rg-np-skip-minor-fgcrg6-74ql4tphgbdm\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61e56fe3-a33c-4f0b-9948-8d6043c67076\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61e56fe3-a33c-4f0b-9948-8d6043c67076\",\n      \"name\": \"61e56fe3-a33c-4f0b-9948-8d6043c67076\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T15:16:34.475724714Z\",\n      \"endTime\": \"2026-10-21T15:40:41.195179605Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj8245n3hvirree84hvsumfles8p8ks still exists (deletion dispatched at 2026-10-21T15:16:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61e56fe3-a33c-4f0b-9948-8d6043c67076\n            correlation ID: cd153f0f-7a7c-424d-b3ee-0368d0322ea3\n            client request ID: 907b5bd2-c9ac-4c9c-8240-9b1e7c8dc931\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0xfdd760f41e0\u003e: \n    resource group \"rg-np-skip-minor-fgcrg6-74ql4tphgbdm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-fgcrg6\" in resourcegroup=\"rg-np-skip-minor-fgcrg6-74ql4tphgbdm\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61e56fe3-a33c-4f0b-9948-8d6043c67076\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61e56fe3-a33c-4f0b-9948-8d6043c67076\",\n      \"name\": \"61e56fe3-a33c-4f0b-9948-8d6043c67076\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T15:16:34.475724714Z\",\n      \"endTime\": \"2026-10-21T15:40:41.195179605Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj8245n3hvirree84hvsumfles8p8ks still exists (deletion dispatched at 2026-10-21T15:16:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/61e56fe3-a33c-4f0b-9948-8d6043c67076\n            correlation ID: cd153f0f-7a7c-424d-b3ee-0368d0322ea3\n            client request ID: 907b5bd2-c9ac-4c9c-8240-9b1e7c8dc931\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"559144b097cee7f893d8d463fe655cd33ae6a02e0c540a445655b2015c8747ea","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0cce093122447f5c3b6f99be8e0de298d1fbee72db77c1703737ae037ec0882b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-08T08:20:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x1e295ecea018\u003e: \n    resource group \"rg-np-noedge-qc2jn7-9vdqhpgwfbsz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-qc2jn7\" in resourcegroup=\"rg-np-noedge-qc2jn7-9vdqhpgwfbsz\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baf0b153-b7d4-4908-8682-0862f2ea6e46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baf0b153-b7d4-4908-8682-0862f2ea6e46\",\n      \"name\": \"baf0b153-b7d4-4908-8682-0862f2ea6e46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:55:07.97017731Z\",\n      \"endTime\": \"2026-10-21T15:19:11.127007148Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj822velhkipn0rdu6sccipp887shou still exists (deletion dispatched at 2026-10-21T14:55:08Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baf0b153-b7d4-4908-8682-0862f2ea6e46\n            correlation ID: 397cd54c-667d-4f35-89d7-acd336d7e091\n            client request ID: 1fdb4833-29c6-49dc-b5c0-cbeda17a4258\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x1e295ecea018\u003e: \n    resource group \"rg-np-noedge-qc2jn7-9vdqhpgwfbsz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-qc2jn7\" in resourcegroup=\"rg-np-noedge-qc2jn7-9vdqhpgwfbsz\" to finish deleting: GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baf0b153-b7d4-4908-8682-0862f2ea6e46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baf0b153-b7d4-4908-8682-0862f2ea6e46\",\n      \"name\": \"baf0b153-b7d4-4908-8682-0862f2ea6e46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T14:55:07.97017731Z\",\n      \"endTime\": \"2026-10-21T15:19:11.127007148Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tj822velhkipn0rdu6sccipp887shou still exists (deletion dispatched at 2026-10-21T14:55:08Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9083904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/baf0b153-b7d4-4908-8682-0862f2ea6e46\n            correlation ID: 397cd54c-667d-4f35-89d7-acd336d7e091\n            client request ID: 1fdb4833-29c6-49dc-b5c0-cbeda17a4258\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"d196d36f3d97139794fb2d79fc0812e66fde6e469dbacf774ceffec8cdf5d5ec","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b9c2ed267bf24bee3ef98ff7d56e123a7d3981d4eeb38cb89632ca27800554a8","failure_pattern":"ERROR CODE: InvalidResource; detail message EtcdEncryptionKMSKeyAccessValidation: KeyVaultNetworkAccessDenied: The control plane cannot reach etcd encryption key \"etcd-data-kms-encryption-key\" in Key Vault \"cust-kv-4j7orvssmvpqc\": Key Vault responded with HTTP 403 (ForbiddenByConnection): Public network access is disabled and request is not from a trusted service nor via an approved private link. Clusters created with etcd.dataEncryption.customerManaged.kms.visibility \"Private\" reach the Key Vault from the virtual network that contains the cluster's VNet integration subnet. Ensure the Key Vault has an approved private endpoint, and that a privatelink.vaultcore.azure.net private DNS zone with a record for the Key Vault is linked to that virtual network.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidResource","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fddd609a460\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-zlfmgtnzzr42\" to finish creating: GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d4c1fe4-fcd1-4c1a-9e7c-e514ce812ccd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d4c1fe4-fcd1-4c1a-9e7c-e514ce812ccd\",\n      \"name\": \"1d4c1fe4-fcd1-4c1a-9e7c-e514ce812ccd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:28:36.459657793Z\",\n      \"endTime\": \"2026-10-21T21:43:30.938568702Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"EtcdEncryptionKMSKeyAccessValidation: KeyVaultNetworkAccessDenied: The control plane cannot reach etcd encryption key \\\"etcd-data-kms-encryption-key\\\" in Key Vault \\\"cust-kv-4j7orvssmvpqc\\\": Key Vault responded with HTTP 403 (ForbiddenByConnection): Public network access is disabled and request is not from a trusted service nor via an approved private link. Clusters created with etcd.dataEncryption.customerManaged.kms.visibility \\\"Private\\\" reach the Key Vault from the virtual network that contains the cluster's VNet integration subnet. Ensure the Key Vault has an approved private endpoint, and that a privatelink.vaultcore.azure.net private DNS zone with a record for the Key Vault is linked to that virtual network.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d4c1fe4-fcd1-4c1a-9e7c-e514ce812ccd\n            correlation ID: f0c1fd73-2e2e-42e9-ba5b-c72a1b7faac0\n            client request ID: 536018dd-5a4e-47e8-879c-f794371f0ffe\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fddd609a460\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-zlfmgtnzzr42\" to finish creating: GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d4c1fe4-fcd1-4c1a-9e7c-e514ce812ccd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidResource\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d4c1fe4-fcd1-4c1a-9e7c-e514ce812ccd\",\n      \"name\": \"1d4c1fe4-fcd1-4c1a-9e7c-e514ce812ccd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T21:28:36.459657793Z\",\n      \"endTime\": \"2026-10-21T21:43:30.938568702Z\",\n      \"error\": {\n        \"code\": \"InvalidResource\",\n        \"message\": \"EtcdEncryptionKMSKeyAccessValidation: KeyVaultNetworkAccessDenied: The control plane cannot reach etcd encryption key \\\"etcd-data-kms-encryption-key\\\" in Key Vault \\\"cust-kv-4j7orvssmvpqc\\\": Key Vault responded with HTTP 403 (ForbiddenByConnection): Public network access is disabled and request is not from a trusted service nor via an approved private link. Clusters created with etcd.dataEncryption.customerManaged.kms.visibility \\\"Private\\\" reach the Key Vault from the virtual network that contains the cluster's VNet integration subnet. Ensure the Key Vault has an approved private endpoint, and that a privatelink.vaultcore.azure.net private DNS zone with a record for the Key Vault is linked to that virtual network.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9033216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1d4c1fe4-fcd1-4c1a-9e7c-e514ce812ccd\n            correlation ID: f0c1fd73-2e2e-42e9-ba5b-c72a1b7faac0\n            client request ID: 536018dd-5a4e-47e8-879c-f794371f0ffe\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7394/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113003926449033216","occurred_at":"2026-10-21T20:26:04Z","signature_id":"ce8edb64d8a09b95f942daeb15ed9803a5adb2fb1fcc796d39525347cf26e206","pr_number":7394,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,1,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:180]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0x27455a542030\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x27455a1f8020\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                err: \u003c*errors.joinError | 0x27455a436c78\u003e{\n                    errs: [\n                        \u003c*errors.errorString | 0x27455ba317e0\u003e{\n                            s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        },\n                    ],\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:180]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0x27455a542030\u003e: \n    VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x27455a1f8020\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                err: \u003c*errors.joinError | 0x27455a436c78\u003e{\n                    errs: [\n                        \u003c*errors.errorString | 0x27455ba317e0\u003e{\n                            s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.129.0.21:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n                        },\n                    ],\n                },\n            },\n        ],\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7655/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112878574095568896","occurred_at":"2026-10-21T12:08:02Z","signature_id":"b6106887a382dd0394dcb0db53cac541a359619b96cbc3ff8351488bb27d83ed","pr_number":7655,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"281d6f2e10296c7900f78767c52e72d480329fd4f2572568eaae28e31a3fe696","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-07T03:35:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n    \u003c*errors.joinError | 0x4c6e807e150\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x4c6e7c1c180\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x4c6e7c1c160\u003e{\n                    msg: \"failed to list services: Get \\\"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x4c6e885e0f0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n                        Err: \u003c*net.OpError | 0x4c6e7fc60f0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x4c6e724a820\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x4c6e7d16300\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x4c6e7d162e0\u003e{\n                    msg: \"failed to list all APIServices: Get \\\"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x4c6e898e330\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n                        Err: \u003c*net.OpError | 0x4c6e724a960\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x4c6e724a820\u003e0x4c6e724a820 (cyclic reference),\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n    \u003c*errors.joinError | 0x4c6e807e150\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/api/v1/namespaces/default/services\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x4c6e7c1c180\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Get \\\"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x4c6e7c1c160\u003e{\n                    msg: \"failed to list services: Get \\\"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/api/v1/namespaces/default/services\\\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x4c6e885e0f0\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/api/v1/namespaces/default/services\",\n                        Err: \u003c*net.OpError | 0x4c6e7fc60f0\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x4c6e724a820\u003e{\n                                UnwrapErr: nil,\n                                Err: \"read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\",\n                                Name: \"api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud\",\n                                Server: \"172.30.0.10:53\",\n                                IsTimeout: true,\n                                IsTemporary: true,\n                                IsNotFound: false,\n                            },\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x4c6e7d16300\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \\\"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\",\n                err: \u003c*fmt.wrapError | 0x4c6e7d162e0\u003e{\n                    msg: \"failed to list all APIServices: Get \\\"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\\\": dial tcp: lookup api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud on 172.30.0.10:53: read udp 172.24.69.56:36903-\u003e172.30.0.10:53: i/o timeout\",\n                    err: \u003c*url.Error | 0x4c6e898e330\u003e{\n                        Op: \"Get\",\n                        URL: \"https://api.t3n3q3u0x5r5i1c.3v9v.j9441920.hcp.osadev.cloud:443/apis/apiregistration.k8s.io/v1/apiservices\",\n                        Err: \u003c*net.OpError | 0x4c6e724a960\u003e{\n                            Op: \"dial\",\n                            Net: \"tcp\",\n                            Source: nil,\n                            Addr: nil,\n                            Err: \u003c*net.DNSError | 0x4c6e724a820\u003e0x4c6e724a820 (cyclic reference),\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112937980879441920","occurred_at":"2026-10-21T16:04:05Z","signature_id":"40422f931ae5f5fa8e1de2aaec953c1c5609a51d397f5204389ab4c7eb8c4e1e","pr_number":7646,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-13T17:32:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:165]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3b1b48589220\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3b1b48589200\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:165]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3b1b48589220\u003e: \n    VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: context deadline exceeded\n    {\n        msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x3b1b48589200\u003e{\n            msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"e08364dca80731f0418c929fe004234f557bfbd421934b3e7f47df4ef78a3883","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"46dc672bed35277c15b0159f387f293fc0e41bbe73c60a38930f56426dfc64a8","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded","search_query":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x268bb264c8a0\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x268bb264c840\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x268bb264c8a0\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x268bb264c840\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113033399156346880","occurred_at":"2026-10-21T22:23:15Z","signature_id":"4fd43733ca395408a2c729a0ede7c32b4d82c4b812079a06d3732d6ba109b6b3","pr_number":7592,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"747a82787873c77f04fe842d4a667465cc7031312226d72afba51f023178a98f","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline","search_query":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24cdc40f4060\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n        err: \u003c*fmt.wrapError | 0x24cdc4962880\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n            err: \u003c*fmt.wrapError | 0x24cdc4962840\u003e{\n                msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n                err: \u003c*errors.errorString | 0x24cdc686cd30\u003e{\n                    s: \"rate: Wait(n=1) would exceed context deadline\",\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x24cdc40f4060\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n        err: \u003c*fmt.wrapError | 0x24cdc4962880\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n            err: \u003c*fmt.wrapError | 0x24cdc4962840\u003e{\n                msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n                err: \u003c*errors.errorString | 0x24cdc686cd30\u003e{\n                    s: \"rate: Wait(n=1) would exceed context deadline\",\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113042472400588800","occurred_at":"2026-10-21T22:59:18Z","signature_id":"63a64e6b0207c4489689dc0e7288074b337a6df55f9172ea9680207d7e5a6bea","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"af6ad342899cd811ce087a1b8b4062889908439af02e46630813eba3a1ef329d","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded","search_query":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x266d4ce3a1e0\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x266d4ce3a180\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x266d4ce3a1e0\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x266d4ce3a180\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7675/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113033770973007872","occurred_at":"2026-10-21T22:24:43Z","signature_id":"173db782edff4a8598adc15560a4c17c6a918f84c919c536a105292016ff8ad0","pr_number":7675,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"440ad8fc83e7a25f7f7b8010c9df7fcbc03c28015d7d75ff986daa1fab0155a3","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.003s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:","search_query":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.003s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x25ba9dd67100\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.003s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\n      - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/ubi9/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.003s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        err: \u003c*errors.errorString | 0x25ba9e656230\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x25ba9dd67100\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.003s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\n      - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/ubi9/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.003s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        err: \u003c*errors.errorString | 0x25ba9e656230\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113041676711759872","occurred_at":"2026-10-21T22:56:08Z","signature_id":"2ce757208155deb3f68d6463ba90393730be81594552edc6efbacb43ba1cfd1c","pr_number":7573,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"c63ca7ee65f760707d7ea6438968e734982a8cef0fb608eebbd7065dc93f8cf0","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.008s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded","search_query":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.008s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x550b080a240\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.008s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.008s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x550b080a180\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x550b080a240\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.008s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.008s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x550b080a180\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7646/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113041621648936960","occurred_at":"2026-10-21T22:55:55Z","signature_id":"5c266d660c202fe4aed13c68abfdf57e8361ad074f4b991cc8a01a1cf54edcf1","pr_number":7646,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc71d272508640666ee62dec8816ec8c0fa480a998e34156d578468c2d17e330","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.024s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:","search_query":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.024s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32bf9ef05a0\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.024s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\n      - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/ubi9/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.024s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        err: \u003c*errors.errorString | 0x32bf9447a70\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32bf9ef05a0\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.024s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\n      - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/ubi9/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.024s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        err: \u003c*errors.errorString | 0x32bf9447a70\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113043630527614976","occurred_at":"2026-10-21T23:03:54Z","signature_id":"2aec020a8bfde6408b3a302282c14c135420e93a232cf877e3f2870576611f90","pr_number":7177,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"80ce72c82f8d7096296ee4caffe80f1ae92c3cec63ab99884ce9c40cf2ed18e9","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded","search_query":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1af71017f080\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1af71017f060\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1af71017f080\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x1af71017f060\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113029875274092544","occurred_at":"2026-10-21T22:09:08Z","signature_id":"755b80f23699f6662cfc709469dd31f6d0c76641d658a0c03af11e7de0995079","pr_number":7641,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":33,"trend_counts":[9,0,0,0,4,9,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":102,"prior_last_seen_at":"2026-10-15T19:10:04Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 12 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:11:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.176:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7od9dtldg9atesqujpab56gdtktc3-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-sxf4w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7od9dtldg9atesqujpab56gdtktc3-sre-hcp-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:11:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.176:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7od9dtldg9atesqujpab56gdtktc3-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-sxf4w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7od9dtldg9atesqujpab56gdtktc3-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:11:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.176:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7od9dtldg9atesqujpab56gdtktc3-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-sxf4w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7od9dtldg9atesqujpab56gdtktc3-sre-hcp-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:11:48Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.176:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7od9dtldg9atesqujpab56gdtktc3-sre-hcp-cluster\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-sxf4w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7od9dtldg9atesqujpab56gdtktc3-sre-hcp-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:20:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.176:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7lluppmfrbdhhdr0g5hdh5ec68bse-m8x0b3m3p3c6a3w\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-sxf4w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7lluppmfrbdhhdr0g5hdh5ec68bse-m8x0b3m3p3c6a3w/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:20:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.176:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7lluppmfrbdhhdr0g5hdh5ec68bse-m8x0b3m3p3c6a3w\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-sxf4w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7lluppmfrbdhhdr0g5hdh5ec68bse-m8x0b3m3p3c6a3w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:22:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.206:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7kg5i5v3qeqbhhu1aj4t26p20km4l-s6i4e5m7h6r7f2g\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-v576p\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7kg5i5v3qeqbhhu1aj4t26p20km4l-s6i4e5m7h6r7f2g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:22:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.206:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7kg5i5v3qeqbhhu1aj4t26p20km4l-s6i4e5m7h6r7f2g\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-v576p\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7kg5i5v3qeqbhhu1aj4t26p20km4l-s6i4e5m7h6r7f2g/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:22:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.206:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7kg5i5v3qeqbhhu1aj4t26p20km4l-s6i4e5m7h6r7f2g\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-v576p\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7kg5i5v3qeqbhhu1aj4t26p20km4l-s6i4e5m7h6r7f2g/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:22:46Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.206:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7kg5i5v3qeqbhhu1aj4t26p20km4l-s6i4e5m7h6r7f2g\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-v576p\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7kg5i5v3qeqbhhu1aj4t26p20km4l-s6i4e5m7h6r7f2g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:23:45Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.176:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7lluppmfrbdhhdr0g5hdh5ec68bse-m8x0b3m3p3c6a3w\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-sxf4w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7lluppmfrbdhhdr0g5hdh5ec68bse-m8x0b3m3p3c6a3w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:23:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9083904-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.176:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", namespace=\"ocm-arohcpci01-2tj7lluppmfrbdhhdr0g5hdh5ec68bse-m8x0b3m3p3c6a3w\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-sxf4w\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tj7lluppmfrbdhhdr0g5hdh5ec68bse-m8x0b3m3p3c6a3w/cluster-api has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"3005821d90629eefac85e9c80720fd62b2aba2f3fa8130fe4fea593d50811cb4","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9c14268f5937655bde9e381e9f7b23c185ab03a3643c5982db2abac6823bde46","failure_pattern":"alert [hcp] userJourneyKubeApiserverAvailability1h5m fired","search_query":"alert [hcp] userJourneyKubeApiserverAvailability1h5m fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyKubeApiserverAvailability1h5m does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert userJourneyKubeApiserverAvailability1h5m does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:38:57Z\n  Severity:    Sev3\n  Labels:      _id=\"e3c51a01-e77e-4b85-8495-f258f44ab7a0\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j9083904-mgmt-2\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", name=\"s6i4e5m7h6r7f2g\", namespace=\"ocm-arohcpci01-2tj7kg5i5v3qeqbhhu1aj4t26p20km4l\", region=\"centralus\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-20-hth2d5sgmqkq/providers/microsoft.redhatopenshift/hcpopenshiftclusters/cluster-candidate-4-20-82k925\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-candidate-4-20-hth2d5sgmqkq/providers/microsoft.redhatopenshift/hcpopenshiftclusters/cluster-candidate-4-20-82k925\nManagement Cluster: ci01-j9083904-mgmt-2\nNamespace: ocm-arohcpci01-2tj7kg5i5v3qeqbhhu1aj4t26p20km4l\n\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T15:42:04Z\n  Severity:    Sev3\n  Labels:      _id=\"001fac7e-54b1-459b-8e12-67588890841e\", alertname=\"userJourneyKubeApiserverAvailability1h5m\", cluster=\"ci01-j9083904-mgmt-1\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9083904/providers/microsoft.monitor/accounts/hcps-j9083904\", name=\"sre-hcp-cluster\", namespace=\"ocm-arohcpci01-2tj7od9dtldg9atesqujpab56gdtktc3\", region=\"centralus\", resource_id=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-9q9nnh4t7882/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster\", severity=\"3\", short_window=\"5m\", slo=\"kas-availability\"\n  Description: Resource ID: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-9q9nnh4t7882/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster\nManagement Cluster: ci01-j9083904-mgmt-1\nNamespace: ocm-arohcpci01-2tj7od9dtldg9atesqujpab56gdtktc3"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112891349849083904","occurred_at":"2026-10-21T12:58:45Z","signature_id":"91fcbde7cb5abaca0304362391362c4923ae99a94cb95a31c974414c40574642","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":19,"prior_last_seen_at":"2026-10-15T19:10:09Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 24 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-22T00:37:19Z\n  Ended:       2026-10-22T00:42:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjg9csqcrisiulhj27orktv3umnod15\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjg9csqcrisiulhj27orktv3umnod15/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-22T00:37:19Z\n  Ended:       2026-10-22T00:42:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjg9csqcrisiulhj27orktv3umnod15\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjg9csqcrisiulhj27orktv3umnod15/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-22T00:53:27Z\n  Ended:       2026-10-22T01:09:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-22T00:53:27Z\n  Ended:       2026-10-22T01:10:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-22T00:53:27Z\n  Ended:       2026-10-22T01:10:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-22T00:53:27Z\n  Ended:       2026-10-22T01:09:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-22T00:53:27Z\n  Ended:       2026-10-22T01:10:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-22T00:53:27Z\n  Ended:       2026-10-22T01:10:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-22T00:55:29Z\n  Ended:       2026-10-22T01:01:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgit1lgtucsgcuhe4ui1l694121mei\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgit1lgtucsgcuhe4ui1l694121mei/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-10-22T00:55:29Z\n  Ended:       2026-10-22T01:02:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgit1lgtucsgcuhe4ui1l694121mei\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgit1lgtucsgcuhe4ui1l694121mei/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-10-22T00:56:27Z\n  Ended:       2026-10-22T01:02:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgit1lgtucsgcuhe4ui1l694121mei\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgit1lgtucsgcuhe4ui1l694121mei/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-10-22T00:56:27Z\n  Ended:       2026-10-22T01:02:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgit1lgtucsgcuhe4ui1l694121mei\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgit1lgtucsgcuhe4ui1l694121mei/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-10-22T00:57:32Z\n  Ended:       2026-10-22T01:07:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-10-22T00:57:32Z\n  Ended:       2026-10-22T01:07:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-10-22T00:57:32Z\n  Ended:       2026-10-22T01:07:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-10-22T00:57:32Z\n  Ended:       2026-10-22T01:08:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-10-22T00:57:32Z\n  Ended:       2026-10-22T01:07:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-10-22T00:57:32Z\n  Ended:       2026-10-22T01:07:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-10-22T00:57:32Z\n  Ended:       2026-10-22T01:06:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-10-22T00:57:32Z\n  Ended:       2026-10-22T01:08:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-10-22T00:57:32Z\n  Ended:       2026-10-22T01:05:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-10-22T00:57:32Z\n  Ended:       2026-10-22T01:08:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-10-22T00:57:32Z\n  Ended:       2026-10-22T01:07:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjtr217jj3vq9vq4ij0qgjg910j8v/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-10-22T00:57:33Z\n  Ended:       2026-10-22T01:08:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2tjgjs0baehlrda8ov4icjda3mdfuseh/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113042472400588800","occurred_at":"2026-10-21T22:59:18Z","signature_id":"45f02d37132c7ca69c4b8d35e9c7d9cc49efd1e1f8b8cd29e0cd1b68493e146b","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-15T19:10:09Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 6 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-22T01:03:32Z\n  Ended:       2026-10-22T01:09:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-22T01:03:32Z\n  Ended:       2026-10-22T01:10:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/config-policy-controller is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-22T01:03:33Z\n  Ended:       2026-10-22T01:10:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/config-policy-controller is not progressing for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-22T01:03:33Z\n  Ended:       2026-10-22T01:10:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/governance-policy-framework is not progressing for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-22T01:03:33Z\n  Ended:       2026-10-22T01:09:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-22T01:03:33Z\n  Ended:       2026-10-22T01:10:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0588800-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0588800/providers/microsoft.monitor/accounts/services-j0588800\", namespace=\"klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-msgjh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2tjgi3mscvh8c352mcr946oq21vref8m/governance-policy-framework is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113042472400588800","occurred_at":"2026-10-21T22:59:18Z","signature_id":"8def531b17e9ee39d9a5caeb57551b79162439cd97304f65aeaeb37731daea25","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[3,2,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-10-05","2026-10-12"],"prior_runs_affected":21,"prior_last_seen_at":"2026-10-16T14:51:17Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T18:12:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j5592832-mgmt-1\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-21T18:12:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j5592832-mgmt-2\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112951578825592832","occurred_at":"2026-10-21T16:58:07Z","signature_id":"a48739b8da6b21a7dfca0bb50093d532345488047d974d8bf9487a45f6a0f7b5","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-z6nkl9f5sbc4\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":138,"prior_last_seen_at":"2026-10-16T16:35:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should serve the default ingress certificate for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x31d9eb504ae0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-z6nkl9f5sbc4\" to finish creating: GET https://rp.j5471360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17f3d6d4-06d1-4780-9e78-cf9749f6ba8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17f3d6d4-06d1-4780-9e78-cf9749f6ba8b\",\n      \"name\": \"17f3d6d4-06d1-4780-9e78-cf9749f6ba8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T18:00:06.904278761Z\",\n      \"endTime\": \"2026-10-21T18:19:13.24922172Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2z3o8a0b5o6d5o-np-1-tgcm2-k8hzl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-z6nkl9f5sbc4--managed/providers/Microsoft.Compute/virtualMachines/v2z3o8a0b5o6d5o-np-1-tgcm2-k8hzl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2z3o8a0b5o6d5o-np-1-tgcm2-k8hzl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-z6nkl9f5sbc4--managed/providers/Microsoft.Compute/virtualMachines/v2z3o8a0b5o6d5o-np-1-tgcm2-k8hzl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5471360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17f3d6d4-06d1-4780-9e78-cf9749f6ba8b\n            correlation ID: cc57308c-c983-4f14-a613-2d5dd8c79cbb\n            client request ID: 38a10c4b-f563-4456-898f-02a08bc824fc\nfail [github.com/Azure/ARO-HCP/test/e2e/default_cluster_ingress_cert.go:119]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x31d9eb504ae0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"default-ingress-cert\" in resourcegroup=\"ingress-cert-le-z6nkl9f5sbc4\" to finish creating: GET https://rp.j5471360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17f3d6d4-06d1-4780-9e78-cf9749f6ba8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17f3d6d4-06d1-4780-9e78-cf9749f6ba8b\",\n      \"name\": \"17f3d6d4-06d1-4780-9e78-cf9749f6ba8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T18:00:06.904278761Z\",\n      \"endTime\": \"2026-10-21T18:19:13.24922172Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2z3o8a0b5o6d5o-np-1-tgcm2-k8hzl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-z6nkl9f5sbc4--managed/providers/Microsoft.Compute/virtualMachines/v2z3o8a0b5o6d5o-np-1-tgcm2-k8hzl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2z3o8a0b5o6d5o-np-1-tgcm2-k8hzl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-z6nkl9f5sbc4--managed/providers/Microsoft.Compute/virtualMachines/v2z3o8a0b5o6d5o-np-1-tgcm2-k8hzl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j5471360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17f3d6d4-06d1-4780-9e78-cf9749f6ba8b\n            correlation ID: cc57308c-c983-4f14-a613-2d5dd8c79cbb\n            client request ID: 38a10c4b-f563-4456-898f-02a08bc824fc"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112945930595471360","occurred_at":"2026-10-21T16:35:41Z","signature_id":"be63ac44558e7a08a8dbec37dc919d8ebf64aea1f85a2a1d7fed6c2008c3ffef","pr_number":7592,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2ba8125ee6d6927daa62a059a4ff043523c4c106539363e3411b67997f537b63","failure_pattern":"failed to list ClusterOperators: context deadline exceeded","search_query":"failed to list ClusterOperators: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0x5f255b9c0d8\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: context deadline exceeded\n    {\n        errs: [\n            \u003ccontext.deadlineExceededError\u003e{},\n            \u003c*fmt.wrapError | 0x5f2544743e0\u003e{\n                msg: \"failed to list ClusterOperators: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:255]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n    \u003c*errors.joinError | 0x5f255b9c0d8\u003e: \n    context deadline exceeded\n    failed to list ClusterOperators: context deadline exceeded\n    {\n        errs: [\n            \u003ccontext.deadlineExceededError\u003e{},\n            \u003c*fmt.wrapError | 0x5f2544743e0\u003e{\n                msg: \"failed to list ClusterOperators: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112870236943290368","occurred_at":"2026-10-21T11:34:40Z","signature_id":"2b31497ba3f351893ae6325dbb9dbe44cc926cbdc9f95f92703278b78d05e485","pr_number":7658,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1600402ac5fa81bdffb2cbc68e8a87faa5f1bf014a325f322a363592b9c546f5","failure_pattern":"failed to roll out Helm release: execution error at (swift-recorder/templates/daemonset.yaml:37:79): swift-recorder requires an image digest when enabled","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: execution error at (swift-recorder/templates/daemonset.yaml:37:79): swift-recorder requires an image digest when enabled","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SwiftRecorder/management/deploy (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SwiftRecorder/management/deploy (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-10-21T22:26:12.219Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-21T22:26:12.557Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 values=\"map[captureMode:all clusterName:ci01-j3775744-mgmt-2 enabled:true environment:ci01 image:map[digest: registry:arohcpsvcdev.azurecr.io repository:swift-recorder] region:canadacentral resources:map[limits:map[memory:256Mi] requests:map[cpu:20m memory:64Mi]] serviceAccount:map[name:swift-recorder]]\"\ntime=2026-10-21T22:26:12.557Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-10-21T22:26:12.728Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 namespace=swift-recorder\ntime=2026-10-21T22:26:12.944Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-21T22:26:12.944Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-10-21T22:26:12.999Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-10-21T22:26:13.152Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"execution error at (swift-recorder/templates/daemonset.yaml:37:79): swift-recorder requires an image digest when enabled\"\ntime=2026-10-21T22:26:13.152Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: execution error at (swift-recorder/templates/daemonset.yaml:37:79): swift-recorder requires an image digest when enabled\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113015450223775744","occurred_at":"2026-10-21T21:11:55Z","signature_id":"bafefa45e1dfbd9aac133aecb4a00c05e25cae616fffc80021ced4e0f8a4f8ca","pr_number":7672,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d12a50d3abbc44ec2f70d4f09519cd251ad5fbd0b773c190bbfff84f2bff0104","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=11048, acm-agent=207","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=11048, acm-agent=207","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2df52e691b0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=11048, acm-agent=207\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=11048, acm-agent=207\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2df52e691b0\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=11048, acm-agent=207\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [maestro]; observed counts: maestro=0, hypershift=11048, acm-agent=207\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112951578825592832","occurred_at":"2026-10-21T16:58:07Z","signature_id":"a6160ff9344a697f421b39fb1c1bec5684dcf1281caeaeeb394912b4cd0b3f60","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b1d68134ee1583cdf3cf29ccf5f52a00bbda8ef7bbf33a93cfb29ce106c7b3bb","failure_pattern":"not all expected service metrics appeared in Azure Monitor","search_query":"not all expected service metrics appeared in Azure Monitor","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from SVC and MGMT clusters to be present in Azure Monitor WOrkspace","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2048192771084338,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-10-15..2026-10-21 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering want metrics from SVC and MGMT clusters to be present in Azure Monitor WOrkspace","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.001s.\nnot all expected service metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:\nexpected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: []\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.001s.\nnot all expected service metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:\nexpected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: []\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2112951578825592832","occurred_at":"2026-10-21T16:58:07Z","signature_id":"253846ad4942964c1651ba10fc67bbc0524142fa2ec1419199960cf6511ba7f7","pr_number":7665,"after_last_push_of_merged_pr":false}]}]}]}