{"meta":{"start_date":"2026-10-18","end_date":"2026-10-18","start_at":"2026-10-18T00:00:00Z","end_at":"2026-10-19T00:00:00Z","timezone":"UTC","generated_at":"2026-10-30T08:09:54Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":11,"failed_runs":7,"raw_occurrences":7,"matched_occurrences":7,"runs_affected":7},"rows":[{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":18.181818181818183,"anchor_occurrences":2,"after_last_push_count":39,"trend_counts":[5,6,4,10,9,1,2],"trend_range":"2026-10-12..2026-10-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28","2026-10-05"],"prior_runs_affected":199,"prior_last_seen_at":"2026-10-11T23:29:37Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":2}],"full_error_samples":["alert fired 38 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-tmw4b\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-tmw4b has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-4fdv6\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-4fdv6 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-5vc4f\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-5vc4f has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-dfx48\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-dfx48 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-tmkfh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-tmkfh has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-ldq7g\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-ldq7g has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-4prhm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-4prhm has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-mtvd8\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-mtvd8 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-w9lfx\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-w9lfx has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-m6kzd\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-m6kzd has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-gf4mg\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-gf4mg has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-jtg79\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-jtg79 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-gbc9d\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-gbc9d has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-vnwdn\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-vnwdn has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-cvdcq\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-cvdcq has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-jp4zq\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-jp4zq has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-lcvwv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-lcvwv has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-nllhh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-nllhh has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-x8lvp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-x8lvp has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-2cp2r\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-2cp2r has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-bnghx\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-bnghx has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-spjrm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-spjrm has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-7gwqf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-7gwqf has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-wzhd4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-wzhd4 has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-z67pp\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-z67pp has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n  State:       Resolved\n  Started:     2026-10-18T22:30:02Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-8rjbk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-8rjbk has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-89fp5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-89fp5 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-dfhsg\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-dfhsg has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-ndv7w\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-ndv7w has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-x65xf\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-x65xf has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-9nhcz\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-9nhcz has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-57rg5\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-57rg5 has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:07Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-vqfkk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-vqfkk has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-mvbzv\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-mvbzv has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-xqzzm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-xqzzm has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-pzz7r\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-pzz7r has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-4ht2v\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-4ht2v has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n  State:       Resolved\n  Started:     2026-10-18T22:31:04Z\n  Ended:       2026-10-18T22:55:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7176064-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b\", pod=\"network-node-identity-6585b9796c-m2n57\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2thfcg8puuokl14unu497m3b3fo4ib0u-q2l3y4m0x2b1n0b/network-node-identity-6585b9796c-m2n57 has been in a non-ready state for longer than 5 minutes.","alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-18T02:22:20Z\n  Ended:       2026-10-18T02:28:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5039232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tgtj1aja7eu3kumpui0v9v0hrn0ctrn-n2t2k5k1d2q7b4m\", pod=\"openshift-apiserver-74747bbdf5-l6sx2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tgtj1aja7eu3kumpui0v9v0hrn0ctrn-n2t2k5k1d2q7b4m/openshift-apiserver-74747bbdf5-l6sx2 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111927716537176064","occurred_at":"2026-10-18T21:09:39Z","signature_id":"e4b3d941e23ccd903b4c4ef9302dc754f91d92aac5112d970448006082f1a4cc","pr_number":7387,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7382/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111625789635039232","occurred_at":"2026-10-18T01:09:54Z","signature_id":"0db548d313b61e121d3be7e10a5b288fe32452c78f9c7de6b86df38a44955827","pr_number":7382,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fbdf4b29275bbcfcba0aba86a244253ee3cdcaa74712c6d2e008d1ca9e4f19bc","failure_pattern":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \"globalMSIId\" not found","search_query":"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":18.181818181818183,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,4,0,2],"trend_range":"2026-10-12..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":2}],"full_error_samples":["time=2026-10-18T15:38:05.575Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-18T15:38:06.914Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found\"","time=2026-10-18T12:10:56.819Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-18T12:10:57.618Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to get input values: variable invalid: failed to get value for input output.globalMSIId: key \\\"globalMSIId\\\" not found\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111837676591124480","occurred_at":"2026-10-18T15:11:52Z","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","pr_number":7580,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111776757118930944","occurred_at":"2026-10-18T11:09:48Z","signature_id":"6c85c0c84a029cc468c08af59c9556ea1e898db311553fdd2c177b18ce147d44","pr_number":7580,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"835eba9e18d4ce34b2c45e7a932a8d1b8ec3d4cc9cdfed55b9778db69913e7ed","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RequestTimeout; detail message Database account creation failed. Operation Id: \u003cuuid\u003e, Error : Message: {\"Errors\":[\"The requested operation exceeded maximum alloted time. Learn more: \u003curl\u003e; provider Microsoft.DocumentDB","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-10-12..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":1}],"full_error_samples":["time=2026-10-18T22:01:09.062Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-10-18T22:01:10.583Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-10-18T22:01:11.299Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d2f2367984b48d226a7d218c1b893f5e8345defcea4da99b6318b3f53d90ed49 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6019200%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd2f2367984b48d226a7d218c1b893f5e8345defcea4da99b6318b3f53d90ed49\ntime=2026-10-18T22:08:43.317Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6019200/providers/Microsoft.Resources/deployments/d2f2367984b48d226a7d218c1b893f5e8345defcea4da99b6318b3f53d90ed49/operationStatuses/08584092460146839094\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6019200/providers/Microsoft.Resources/deployments/rp-cosmos-account\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6019200/providers/Microsoft.DocumentDB/databaseAccounts/arohcpci01-rp-j6019200\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"RequestTimeout\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Database account creation failed. Operation Id: 29cb1d84-68fc-41bc-85ce-f7a058c770ca, Error : Message: {\\\\\\\\\\\\\\\"Errors\\\\\\\\\\\\\\\":[\\\\\\\\\\\\\\\"The requested operation exceeded maximum alloted time. Learn more: https://aka.ms/cosmosdb-tsg-service-request-timeout\\\\\\\\\\\\\\\"]}\\\\\\\\r\\\\\\\\nActivityId: 510ce633-a87c-4347-9741-f3200e8ace7f, Request URI: /apps/afdfb817-7910-4985-b158-590770c1c1bf/services/f1fdfe14-2eef-4dca-b710-c2c6e8884c3f/partitions/918f52c2-0d35-4150-826f-7edd51cc07ef/replicas/134367860023629100s, RequestStats: \\\\\\\\r\\\\\\\\nRequestStartTime: 2026-10-18T22:07:00.4462358Z, RequestEndTime: 2026-10-18T22:07:05.4461411Z,  Number of regions attempted:1\\\\\\\\r\\\\\\\\n{\\\\\\\\\\\\\\\"systemHistory\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:06:12.0209020Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.273,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20915668.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0851,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32764,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:06:22.0359077Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.546,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20914336.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0463,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:06:32.0513471Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.449,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20922484.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0842,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:06:42.0523437Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.762,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20935348.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.083,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:06:52.0699478Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.390,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20926436.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0518,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32},{\\\\\\\\\\\\\\\"dateUtc\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:07:02.0711735Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"cpu\\\\\\\\\\\\\\\":0.332,\\\\\\\\\\\\\\\"memory\\\\\\\\\\\\\\\":20925148.000,\\\\\\\\\\\\\\\"threadInfo\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"isThreadStarving\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"threadWaitIntervalInMs\\\\\\\\\\\\\\\":0.0848,\\\\\\\\\\\\\\\"availableThreads\\\\\\\\\\\\\\\":32765,\\\\\\\\\\\\\\\"minThreads\\\\\\\\\\\\\\\":8,\\\\\\\\\\\\\\\"maxThreads\\\\\\\\\\\\\\\":32767},\\\\\\\\\\\\\\\"numberOfOpenTcpConnection\\\\\\\\\\\\\\\":32}]}\\\\\\\\r\\\\\\\\nRequestStart: 2026-10-18T22:07:00.4462358Z; ResponseTime: 2026-10-18T22:07:05.4461411Z; StoreResult: StorePhysicalAddress: rntbd://cdb-ms-prod-centralus1-be424.documents.azure.com:11907/apps/afdfb817-7910-4985-b158-590770c1c1bf/services/f1fdfe14-2eef-4dca-b710-c2c6e8884c3f/partitions/918f52c2-0d35-4150-826f-7edd51cc07ef/replicas/134367860023629100s, LSN: -1, GlobalCommittedLsn: -1, PartitionKeyRangeId: , IsValid: False, StatusCode: 408, SubStatusCode: 0, RequestCharge: 0, ItemLSN: -1, SessionToken: , UsingLocalLSN: False, TransportException: null, BELatencyMs: 4996.861, ActivityId: 510ce633-a87c-4347-9741-f3200e8ace7f, RetryAfterInMs: , globalNRegionCommittedGLSN: -1, ReplicaHealthStatuses: [(port: 11907 | status: Connected | lkt: 10/18/2026 10:07:05 PM)], TransportRequestTimeline: {\\\\\\\\\\\\\\\"requestTimeline\\\\\\\\\\\\\\\":[{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Created\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:00.4462358Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0024},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ChannelAcquisitionStarted\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:00.4462382Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0078},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Pipelined\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:00.4462460Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0881},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Transit Time\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:00.4463341Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 4997.4559},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Received\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:05.4437900Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0.0577},{\\\\\\\\\\\\\\\"event\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Completed\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"startTimeUtc\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2026-10-18T22:07:05.4438477Z\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"durationInMs\\\\\\\\\\\\\\\": 0}],\\\\\\\\\\\\\\\"serviceEndpointStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"inflightRequests\\\\\\\\\\\\\\\":1,\\\\\\\\\\\\\\\"openConnections\\\\\\\\\\\\\\\":1},\\\\\\\\\\\\\\\"connectionStats\\\\\\\\\\\\\\\":{\\\\\\\\\\\\\\\"waitforConnectionInit\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"False\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"callsPendingReceive\\\\\\\\\\\\\\\":0,\\\\\\\\\\\\\\\"lastSendAttempt\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:04:46.4616163Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastSend\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:04:46.4616163Z\\\\\\\\\\\\\\\",\\\\\\\\\\\\\\\"lastReceive\\\\\\\\\\\\\\\":\\\\\\\\\\\\\\\"2026-10-18T22:04:52.4460367Z\\\\\\\\\\\\\\\"},\\\\\\\\\\\\\\\"requestSizeInBytes\\\\\\\\\\\\\\\":867,\\\\\\\\\\\\\\\"responseMetadataSizeInBytes\\\\\\\\\\\\\\\":30,\\\\\\\\\\\\\\\"responseBodySizeInBytes\\\\\\\\\\\\\\\":133};\\\\\\\\r\\\\\\\\n ResourceType: Replica, OperationType: ForceConfigRefresh\\\\\\\\r\\\\\\\\n, SDK: Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7592/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111935962996019200","occurred_at":"2026-10-18T21:42:26Z","signature_id":"ddf39a2ec9890a007519b2cb9c2c9eea035cd46a318bec7e09e31e3024b6f696","pr_number":7592,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"69e9a625fc3574b27da6a100519b2f81973de7aebd2a1ccd425027aa08783b72","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-12..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xadaa6040100\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0xadaa51438f0\u003e{\n            s: \"dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xadaa6040100\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-v9j4v95cl9wx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0xadaa51438f0\u003e{\n            s: \"dev/canadacentral/ci01-j1046272-mgmt-1/ocm-arohcpci01-2th7q5a54cq5jldof0f3l7iofpg7u8al-h1p2a3c0s8o4u6j: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7591/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111797770821046272","occurred_at":"2026-10-18T12:33:18Z","signature_id":"b43432c9ea207ac983c242c108e787e684ab34e3936a86b9df05974662681ba9","pr_number":7591,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-10-12..2026-10-18 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":1}],"full_error_samples":["time=2026-10-18T21:53:08.675Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: IstioUpgrade\\n  AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-18T21:53:10.010Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3670528-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-18T21:53:10.010Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3670528-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-18T21:53:10.816Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3670528-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-18T21:53:10.816Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j3670528-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-18T21:53:10.816Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j3670528-svc\ntime=2026-10-18T22:08:13.794Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2111930721403670528","occurred_at":"2026-10-18T21:21:36Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7580,"after_last_push_of_merged_pr":true}]}]}]}