{"meta":{"start_date":"2026-10-17","end_date":"2026-10-17","start_at":"2026-10-17T00:00:00Z","end_at":"2026-10-18T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T07:01:36Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":3,"matched_occurrences":3,"runs_affected":2},"rows":[{"environment":"dev","failure_pattern_id":"58fc1cbd5bf08c0e9cacc04febc66fa47a6ea8ac19d3acc5b10cdd7a31b22169","failure_pattern":"RevokeCredentialsAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-mjx8hf\" in resourcegroup=\"admin-credential-lifecycle-test-k6wptwmmb252\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-11..2026-10-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: failed to revoke admin credentials for cluster \"admin-cred-lifecycle-mjx8hf\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1ab46da90510\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-mjx8hf\" in resourcegroup=\"admin-credential-lifecycle-test-k6wptwmmb252\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-mjx8hf\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-k6wptwmmb252\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1ab46cac9c40\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: failed to revoke admin credentials for cluster \"admin-cred-lifecycle-mjx8hf\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x1ab46da90510\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-mjx8hf\" in resourcegroup=\"admin-credential-lifecycle-test-k6wptwmmb252\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-mjx8hf\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-k6wptwmmb252\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x1ab46cac9c40\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-mjx8hf in resource group admin-credential-lifecycle-test-k6wptwmmb252\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111378042730319872","occurred_at":"2026-10-17T08:45:27Z","signature_id":"70abbc638303e2bdc13f44e1dfd8ef96ce569ae304e7f4344645cf766995fb90","pr_number":7250,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":37,"trend_counts":[4,5,6,4,10,9,1],"trend_range":"2026-10-11..2026-10-17 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":197,"prior_last_seen_at":"2026-10-11T23:29:37Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 32 time(s) (25 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-17T09:53:56Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w\", pod=\"network-node-identity-5479889bd6-wm9n5\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w/network-node-identity-5479889bd6-wm9n5 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-17T09:53:56Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w\", pod=\"ovnkube-control-plane-6f598f876b-4lwmh\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgf9u7qesjgmttof4tbj175kd9c3v6a-j2l3z7n5l1o3m5w/ovnkube-control-plane-6f598f876b-4lwmh has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-17T09:53:57Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa0b8h25qc9a5i20tff4l0fg95ohb-x1q7f4j4x4u2t1g\", pod=\"network-node-identity-5d7f9f545c-9fj88\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa0b8h25qc9a5i20tff4l0fg95ohb-x1q7f4j4x4u2t1g/network-node-identity-5d7f9f545c-9fj88 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-17T09:56:00Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f\", pod=\"ovnkube-control-plane-6776f795c7-pnvf4\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f/ovnkube-control-plane-6776f795c7-pnvf4 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-17T09:56:00Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f\", pod=\"network-node-identity-6484b49989-tpf9j\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa3i9d0dtsn3mqgm2m0jvbj0gost6-c8m7l6f8p4l9x4f/network-node-identity-6484b49989-tpf9j has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-17T09:56:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d\", pod=\"network-node-identity-77ddc9bb4c-pbwr7\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d/network-node-identity-77ddc9bb4c-pbwr7 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-17T09:56:01Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa5d10e98e3mrvr8mdtggsfec8ur7-z9i7c7u7w1a3k9w\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa5d10e98e3mrvr8mdtggsfec8ur7-z9i7c7u7w1a3k9w/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-17T09:56:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d\", pod=\"ovnkube-control-plane-6bd64ff8dc-l5xp2\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacm1jgb7lcrs9ivlei7ps8sq33bk-w6k4h3j3b8i6v3d/ovnkube-control-plane-6bd64ff8dc-l5xp2 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-17T09:58:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w\", pod=\"kube-apiserver-9875495b7-tqsz4\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w/kube-apiserver-9875495b7-tqsz4 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-10-17T09:58:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w\", pod=\"kube-apiserver-9875495b7-222sx\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w/kube-apiserver-9875495b7-222sx has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-10-17T09:58:01Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfadvi8jv413ort809d0n4kq5kno22-s2q4c0i7f2c0r2x\", pod=\"network-node-identity-5d8fccd895-ptmvr\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfadvi8jv413ort809d0n4kq5kno22-s2q4c0i7f2c0r2x/network-node-identity-5d8fccd895-ptmvr has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-10-17T09:58:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacmauure3rehr0ll3kf67lkonqn4-l7q9g3b3j1r5c8t\", pod=\"network-node-identity-86d7bc77f4-zzxmb\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacmauure3rehr0ll3kf67lkonqn4-l7q9g3b3j1r5c8t/network-node-identity-86d7bc77f4-zzxmb has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-10-17T09:58:01Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w\", pod=\"kube-apiserver-9875495b7-scjqk\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacmm0dlphe6vtc50mkmtd3jb2p8q-f2o6b4o3v3g6y5w/kube-apiserver-9875495b7-scjqk has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k\", pod=\"kube-apiserver-5844c8889f-cbz2x\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k/kube-apiserver-5844c8889f-cbz2x has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k\", pod=\"kube-apiserver-5844c8889f-d6fvz\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k/kube-apiserver-5844c8889f-d6fvz has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas\", pod=\"kube-apiserver-8544f679b7-zl64m\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas/kube-apiserver-8544f679b7-zl64m has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa70s9eurd03qajc81h11klkgrmcp-q6y7b0q2i4d5g1m\", pod=\"network-node-identity-6cd587db9f-t4z8h\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa70s9eurd03qajc81h11klkgrmcp-q6y7b0q2i4d5g1m/network-node-identity-6cd587db9f-t4z8h has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas\", pod=\"kube-apiserver-8544f679b7-5fcvk\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas/kube-apiserver-8544f679b7-5fcvk has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c\", pod=\"network-node-identity-5d97d9b5cb-4k88v\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c/network-node-identity-5d97d9b5cb-4k88v has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacn5q7o0r4p4o2djer1ifh59adg8-ea-listq25ppj\", pod=\"network-node-identity-74f67fb4bf-ngwc7\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacn5q7o0r4p4o2djer1ifh59adg8-ea-listq25ppj/network-node-identity-74f67fb4bf-ngwc7 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k\", pod=\"kube-apiserver-5844c8889f-b42qs\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfacoq552p37e3lj875ff1g01eve2p-j5s8a2u8s3o2v6k/kube-apiserver-5844c8889f-b42qs has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfapf0dhnoj0q238ct23l9vk8nc5at-k7b2q6t5f3g5p6n\", pod=\"network-node-identity-fb54b4d4d-v8wgh\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfapf0dhnoj0q238ct23l9vk8nc5at-k7b2q6t5f3g5p6n/network-node-identity-fb54b4d4d-v8wgh has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-10-17T09:58:58Z\n  Ended:       2026-10-17T10:04:58Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c\", pod=\"ovnkube-control-plane-6895bcbdc6-c9kvm\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfadpgoo9rvetrl85b24o6ppv4j0fo-o3i3s1q6c0r8x5c/ovnkube-control-plane-6895bcbdc6-c9kvm has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-10-17T09:58:59Z\n  Ended:       2026-10-17T10:05:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas\", pod=\"kube-apiserver-8544f679b7-q8rkz\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa3iukddj4rbptbdr63ko16fjd5a0-private-kas/kube-apiserver-8544f679b7-q8rkz has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Resolved\n  Started:     2026-10-17T09:59:55Z\n  Ended:       2026-10-17T10:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j0319872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgfa1d4f46ij8t9ghuvhfoa11l51bep-basic-cluster\", pod=\"network-node-identity-5774b9669-l5ggc\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgfa1d4f46ij8t9ghuvhfoa11l51bep-basic-cluster/network-node-identity-5774b9669-l5ggc has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111378042730319872","occurred_at":"2026-10-17T08:45:27Z","signature_id":"89c30c74c5d96251bd10b1bd76400fc32e088893bb54798f40173fdb211c4066","pr_number":7250,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fd8fe2a3f5cde763aa4cd710e5137511c71621f0646cd65805b79189b1d21ac8","failure_pattern":"| take 101, Post \"\u003curl\u003e\": context deadline exceeded","search_query":"| take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":17,"trend_counts":[0,0,2,26,18,10,1],"trend_range":"2026-10-11..2026-10-17 UTC","prior_windows_present":0,"prior_runs_affected":57,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35a206020900\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x35a2060208e0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x35a2051a8cd0\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x35a2060208c0\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x35a206b44300\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35a206020900\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\n    let desires = cosmosResourceSnapshots\n    | where resourceID startswith strcat(target, \"/\")\n    | where resourceType =~ \"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\"\n    | extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\n    | where isempty(content.deletionTimestamp)\n    | extend manifest = content.properties.status.kubeContent\n    | where manifest.kind == \"HostedCluster\" and isempty(manifest.metadata.deletionTimestamp)\n    | extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\n        managementCluster = tolower(tostring(content.properties.spec.managementCluster))\n    | where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\n    | project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \"-\", hostedClusterName);\n    let fleet = cosmosResourceSnapshots\n    | where resourceType =~ \"microsoft.redhatopenshift/stamps/managementclusters\"\n    | extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\n    | join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\n    | summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\n    | summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\n    | where isempty(content.deletionTimestamp)\n    | extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \"/\")[-1])\n    | where isnotempty(clusterName)\n    | project environment, region, sourceCluster, managementCluster, cluster = clusterName;\n    let hcp = desires\n    | join kind=inner fleet on environment, region, sourceCluster, managementCluster\n    | distinct environment, region, cluster, namespace;\n    let inventory = materialize(kubernetesResourceSnapshots\n    | where timestamp \u003e= start and objectKind in (\"Pod\", \"Deployment\")\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\n    | where event != \"Delete\" and isempty(object.metadata.deletionTimestamp));\n    let pods = inventory\n    | where objectKind == \"Pod\" and object.metadata.labels.app == \"private-router\" and object.status.phase == \"Running\"\n    | project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\n    let counts = pods | summarize Current = count() by environment, region, cluster, namespace;\n    let deployments = inventory\n    | where objectKind == \"Deployment\" and name == \"router\"\n    | project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\n    let summaries = containerLogs\n    | where timestamp \u003e= start and container_name == \"swift-recorder\"\n    | where log.controller_name == \"swift-router-check\" and log.field == \"summary\"\n    | extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\n    | lookup kind=inner hcp on environment, region, cluster, namespace\n    | summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\n    | project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\n        SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\n    hcp\n    | join kind=leftouter deployments on environment, region, cluster, namespace\n    | join kind=leftouter counts on environment, region, cluster, namespace\n    | join kind=leftouter pods on environment, region, cluster, namespace\n    | join kind=leftouter summaries on environment, region, cluster, namespace, UID\n    | project Scope = strcat(environment, \"/\", region, \"/\", cluster, \"/\", namespace),\n        Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\n    | order by Scope asc, UID asc\n    | take 101, Post \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\": context deadline exceeded\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x35a2060208e0\u003e{\n            msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n            err: \u003c*errors.Error | 0x35a2051a8cd0\u003e{\n                Op: 1,\n                Kind: 7,\n                Err: \u003c*fmt.wrapError | 0x35a2060208c0\u003e{\n                    msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-x6nbt9kxc5r4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-17T22:37:32.6448555Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n    managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n    Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                    err: \u003c*url.Error | 0x35a206b44300\u003e{\n                        Op: \"Post\",\n                        URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                        Err: \u003ccontext.deadlineExceededError\u003e{},\n                    },\n                },\n                restErrMsg: nil,\n                decoded: nil,\n                permanent: false,\n                inner: nil,\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111575207289294848","occurred_at":"2026-10-17T21:48:55Z","signature_id":"223637359840624c4e713b2fd6730d870d033cda522f1c6b3d937aab39c513d1","pr_number":7382,"after_last_push_of_merged_pr":true}]}]}]}