{"meta":{"date":"2026-10-03","timezone":"UTC","generated_at":"2026-10-31T08:32:08Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":39,"failed_runs":27,"runs_with_occurrences":27,"runs_with_failure_pattern_matches":26,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":12,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106522529362350080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7332,"pr_state":"open","pr_sha":"9d3908a862dc12dea2ae6969184167ee3f53c194","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T23:11:22Z","started_at":"2026-10-03T23:11:22Z","completed_at":"2026-10-04T01:44:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-04T01:51:07.673019618Z","timing_metadata_checked_at":"2026-10-04T01:51:07.673019618Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-04T01:51:07.680543294Z","region_metadata_checked_at":"2026-10-04T01:51:07.680543294Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"07154007428468f20f1112009c5fb6c98549cc6d2bceda73a68d9d99e31f329e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106522529362350080","occurred_at":"2026-10-03T23:11:22Z","failed_at":"e2e","signature_id":"501495b0aef70c6325fa26b9f0b745d7b4ce463804d55b753046dc2f9f5ed6e4","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f72d213c320\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-n6bsrg\" in resourcegroup=\"rg-np-noedge-n6bsrg-vxj5p9qrl42z\" to finish creating: GET https://rp.j2350080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\",\n \"name\": \"83d90b9f-3285-4912-9884-40a78690862b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-04T00:37:10.755457782Z\",\n \"endTime\": \"2026-10-04T00:56:17.299103912Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f72d213c320\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-n6bsrg\" in resourcegroup=\"rg-np-noedge-n6bsrg-vxj5p9qrl42z\" to finish creating: GET https://rp.j2350080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\",\n \"name\": \"83d90b9f-3285-4912-9884-40a78690862b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-04T00:37:10.755457782Z\",\n \"endTime\": \"2026-10-04T00:56:17.299103912Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106519157846577152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7333,"pr_state":"closed","pr_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","final_merged_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T22:57:59Z","started_at":"2026-10-03T22:57:59Z","completed_at":"2026-10-04T01:26:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-04T01:29:47.036933122Z","timing_metadata_checked_at":"2026-10-04T01:29:47.036933122Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-04T01:29:47.040246812Z","region_metadata_checked_at":"2026-10-04T01:29:47.040246812Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106498996229902336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7338,"pr_state":"closed","pr_sha":"e49406545480a4617927868edca9ac3c5a34842c","final_merged_sha":"e49406545480a4617927868edca9ac3c5a34842c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T21:37:52Z","started_at":"2026-10-03T21:37:52Z","completed_at":"2026-10-04T00:08:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-04T00:15:44.473924569Z","timing_metadata_checked_at":"2026-10-04T00:15:44.473924569Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-04T00:15:44.47677176Z","region_metadata_checked_at":"2026-10-04T00:15:44.47677176Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106489060980690944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7336,"pr_state":"closed","pr_sha":"0c82ea710333dd869bbd1a24686f11e37cfe9b99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T20:58:23Z","started_at":"2026-10-03T20:58:23Z","completed_at":"2026-10-03T23:26:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T23:44:22.460109596Z","timing_metadata_checked_at":"2026-10-03T23:44:22.460109596Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T23:44:22.521655699Z","region_metadata_checked_at":"2026-10-03T23:44:22.521655699Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"1cffcfd8e63a16fab06bc543d2dddcc4d75a7d0783b479870d6a4a474731e2cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106489060980690944","occurred_at":"2026-10-03T20:58:23Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T22:09:35Z\n Ended: 2026-10-03T22:15:33Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0690944-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7341/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106478441103626240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7341,"pr_state":"closed","pr_sha":"b870cb897e3705bbdc0b2ea937f9ccc37eee523e","final_merged_sha":"b870cb897e3705bbdc0b2ea937f9ccc37eee523e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T20:15:49Z","started_at":"2026-10-03T20:15:49Z","completed_at":"2026-10-03T22:37:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T22:52:24.373501246Z","timing_metadata_checked_at":"2026-10-03T22:52:24.373501246Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T22:52:24.376685635Z","region_metadata_checked_at":"2026-10-03T22:52:24.376685635Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106473224001818624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7331,"pr_state":"closed","pr_sha":"53a4ed21d7b546b4f4ac2fb406d7d9bec086289b","final_merged_sha":"d5ee24cc09296f5a16b33a9dbe4bd808f7e62ec2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T19:55:27Z","started_at":"2026-10-03T19:55:27Z","completed_at":"2026-10-03T22:23:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T22:31:19.620134079Z","timing_metadata_checked_at":"2026-10-03T22:31:19.620134079Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T22:31:19.639654719Z","region_metadata_checked_at":"2026-10-03T22:31:19.639654719Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"7ab559cff011f1b66c12ed891d2484e47a330df2be2259018021d4f21d3aef1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106473224001818624","occurred_at":"2026-10-03T19:55:27Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T21:07:34Z\n Ended: 2026-10-03T21:13:37Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1818624-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106459631151747072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7332,"pr_state":"open","pr_sha":"8ed2be90842084d1da211b08e7ba64d1a9ba3c4a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T19:01:06Z","started_at":"2026-10-03T19:01:06Z","completed_at":"2026-10-03T21:29:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T21:35:17.526106349Z","timing_metadata_checked_at":"2026-10-03T21:35:17.526106349Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T21:35:17.529442839Z","region_metadata_checked_at":"2026-10-03T21:35:17.529442839Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106458234737922048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7333,"pr_state":"closed","pr_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","final_merged_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T18:55:53Z","started_at":"2026-10-03T18:55:53Z","completed_at":"2026-10-03T21:30:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T21:35:17.745416578Z","timing_metadata_checked_at":"2026-10-03T21:35:17.745416578Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T21:35:17.748356869Z","region_metadata_checked_at":"2026-10-03T21:35:17.748356869Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e92de00c355c6d1e81532aaf6a4b73f81fd2296ad8bab01ff3068380ee421fc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106458234737922048","occurred_at":"2026-10-03T18:55:53Z","failed_at":"e2e","signature_id":"79b90ae5312ada738ac4440b4501fdd97f482ae1f84f6796798dab482e44cbd8","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3528a4bdeac0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vljzr5rw54g9\" to finish creating: GET https://rp.j7922048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"name\": \"336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T20:19:49.133571808Z\",\n \"endTime\": \"2026-10-03T20:38:58.66181942Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 3 of 3 machines are not healthy\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 3 of 3 machines are not ready\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3528a4bdeac0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vljzr5rw54g9\" to finish creating: GET https://rp.j7922048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"name\": \"336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T20:19:49.133571808Z\",\n \"endTime\": \"2026-10-03T20:38:58.66181942Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 3 of 3 machines are not healthy\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 3 of 3 machines are not ready\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106455133536653312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7340,"pr_state":"open","pr_sha":"e0e2b9b5f45257ab5de133a4651e3a7e85f559f0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T18:43:34Z","started_at":"2026-10-03T18:43:34Z","completed_at":"2026-10-03T21:20:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T21:25:11.045254184Z","timing_metadata_checked_at":"2026-10-03T21:25:11.045254184Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T21:25:11.049137372Z","region_metadata_checked_at":"2026-10-03T21:25:11.049137372Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106455066687836160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7338,"pr_state":"closed","pr_sha":"e49406545480a4617927868edca9ac3c5a34842c","final_merged_sha":"e49406545480a4617927868edca9ac3c5a34842c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T18:43:18Z","started_at":"2026-10-03T18:43:18Z","completed_at":"2026-10-03T21:06:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T21:14:13.230244916Z","timing_metadata_checked_at":"2026-10-03T21:14:13.230244916Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T21:14:13.233906904Z","region_metadata_checked_at":"2026-10-03T21:14:13.233906904Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a97fadc90aeb65076d289bf4b43a4b8b8ca1b1766a94e971dbd614fba091911b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106455066687836160","occurred_at":"2026-10-03T18:43:18Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T19:53:25Z\n Ended: 2026-10-03T19:59:26Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7836160-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106451370914615296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7325,"pr_state":"closed","pr_sha":"db96e7598ff60f6d48668a2a08400759438dbc38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T18:28:37Z","started_at":"2026-10-03T18:28:37Z","completed_at":"2026-10-03T20:57:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T21:03:35.174982068Z","timing_metadata_checked_at":"2026-10-03T21:03:35.174982068Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T21:03:35.18103285Z","region_metadata_checked_at":"2026-10-03T21:03:35.18103285Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428533512343552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7326,"pr_state":"closed","pr_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","final_merged_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T16:57:52Z","started_at":"2026-10-03T16:57:52Z","completed_at":"2026-10-03T19:47:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T20:01:11.353963696Z","timing_metadata_checked_at":"2026-10-03T20:01:11.353963696Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T20:01:11.356680087Z","region_metadata_checked_at":"2026-10-03T20:01:11.356680087Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428486020239360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7336,"pr_state":"closed","pr_sha":"0c82ea710333dd869bbd1a24686f11e37cfe9b99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T16:57:41Z","started_at":"2026-10-03T16:57:41Z","completed_at":"2026-10-03T19:45:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T19:51:09.360703987Z","timing_metadata_checked_at":"2026-10-03T19:51:09.360703987Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T19:51:09.363181579Z","region_metadata_checked_at":"2026-10-03T19:51:09.363181579Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"57308dc196e9b749fed5f9c99022a857683b2f3b413fdc447d1728eb8495a540","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428486020239360","occurred_at":"2026-10-03T16:57:41Z","failed_at":"e2e","signature_id":"09d3516d5937c201911e513dc777964c08f8d8e605d4d938d29eb1e35d21ff64","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1590b4a34e00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-hd5zgd7fw6ql\" to finish creating: GET https://rp.j0239360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"name\": \"0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T18:22:53.322060806Z\",\n \"endTime\": \"2026-10-03T18:41:55.898243278Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkgjk-bhjgr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkgjk-bhjgr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1590b4a34e00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-hd5zgd7fw6ql\" to finish creating: GET https://rp.j0239360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"name\": \"0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T18:22:53.322060806Z\",\n \"endTime\": \"2026-10-03T18:41:55.898243278Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkgjk-bhjgr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkgjk-bhjgr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428469284966400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7335,"pr_state":"closed","pr_sha":"b25f2cbc29ff36f009c24cff5cd46c29b5a264ab","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T16:57:37Z","started_at":"2026-10-03T16:57:37Z","completed_at":"2026-10-03T19:37:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T19:51:09.12188513Z","timing_metadata_checked_at":"2026-10-03T19:51:09.12188513Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T19:51:09.124214823Z","region_metadata_checked_at":"2026-10-03T19:51:09.124214823Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428464000143360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7331,"pr_state":"closed","pr_sha":"53a4ed21d7b546b4f4ac2fb406d7d9bec086289b","final_merged_sha":"d5ee24cc09296f5a16b33a9dbe4bd808f7e62ec2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T16:57:35Z","started_at":"2026-10-03T16:57:35Z","completed_at":"2026-10-03T19:40:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T19:51:09.028565921Z","timing_metadata_checked_at":"2026-10-03T19:51:09.028565921Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T19:51:09.031641011Z","region_metadata_checked_at":"2026-10-03T19:51:09.031641011Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"b2d1210b2af6cff0965edc302a3f5d3cf983f0060e1eae9ab8f9bdca8743f586","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428464000143360","occurred_at":"2026-10-03T16:57:35Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T18:23:19Z\n Ended: 2026-10-03T18:38:18Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0143360-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"d894c32d0218011a1977b9caf03d11f629e802277e3530ff3c23bca403be3fde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428464000143360","occurred_at":"2026-10-03T16:57:35Z","failed_at":"alert","signature_id":"fbbc2741d1268c4182e25737f2913295ff9bf11b2acf044ba433a4c9be14b821","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-file-csi-driver-operator-56bd98ff47-n9mw7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-file-csi-driver-operator-56bd98ff47-n9mw7 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-disk-csi-driver-operator-68446dbd4d-gp5r7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-disk-csi-driver-operator-68446dbd4d-gp5r7 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-disk-csi-driver-operator-68446dbd4d-72x46\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-disk-csi-driver-operator-68446dbd4d-72x46 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-disk-csi-driver-operator-68446dbd4d-l8w9v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-disk-csi-driver-operator-68446dbd4d-l8w9v has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-file-csi-driver-operator-56bd98ff47-lhdbd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-file-csi-driver-operator-56bd98ff47-lhdbd has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106422169931091968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7333,"pr_state":"closed","pr_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","final_merged_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T16:32:35Z","started_at":"2026-10-03T16:32:35Z","completed_at":"2026-10-03T18:47:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T18:59:24.668972577Z","timing_metadata_checked_at":"2026-10-03T18:59:24.668972577Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T18:59:24.672386065Z","region_metadata_checked_at":"2026-10-03T18:59:24.672386065Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"95ba3558ad9d83f64fca97491bc033a5a7b9f82bc6e5448545703a90d8923568","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106422169931091968","occurred_at":"2026-10-03T16:32:35Z","failed_at":"e2e","signature_id":"a7a5e0e36a34359f8084193c2d0faab4839b075dfe3e59cc885f6fb7a3743b3d","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xd9700701640\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l9xkxj2bpszs\" to finish creating: GET https://rp.j1091968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"name\": \"d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T17:56:20.630751478Z\",\n \"endTime\": \"2026-10-03T18:15:20.975618436Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xd9700701640\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l9xkxj2bpszs\" to finish creating: GET https://rp.j1091968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"name\": \"d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T17:56:20.630751478Z\",\n \"endTime\": \"2026-10-03T18:15:20.975618436Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l9xkxj2bpszs\" to finish creating: GET https://rp"}},{"row_id":"cdfd22a749c219a348e8d80d869b67c403a54500cebd607e6fd6668db12ff6b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106422169931091968","occurred_at":"2026-10-03T16:32:35Z","failed_at":"alert","signature_id":"d03789470daaa6e20f0e10af6ab181b291bf54e6bdce46582adbae8892beb283","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 69 time(s) (63 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-52l2j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-52l2j has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-vdz8t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-vdz8t has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-k2787\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-k2787 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-zp9nc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-zp9nc has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-vk8sg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-vk8sg has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-fvs76\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-fvs76 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-llstd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-llstd has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-5mcng\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-5mcng has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9sqdg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9sqdg has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:51:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-btfk4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-btfk4 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-dq2vz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-dq2vz has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-x6wz7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-x6wz7 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-ctpt9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-ctpt9 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-j2d9z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-j2d9z has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-4h79j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-4h79j has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-z2w4b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-z2w4b has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-nkt4w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-nkt4w has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-ffhbr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-ffhbr has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-csvsv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-csvsv has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-s4h6v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-s4h6v has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-dl5l4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-dl5l4 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-47862\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-47862 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-gw8rm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-gw8rm has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-mrcs5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-mrcs5 has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-tzs9x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-tzs9x has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-pjk6c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-pjk6c has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-vdtmc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-vdtmc has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-7kmn4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-7kmn4 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-6w4zc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-6w4zc has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-mnfxd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-mnfxd has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-m84qg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-m84qg has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-j8kmn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-j8kmn has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-r827x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-r827x has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lscgk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lscgk has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-gj4v6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-gj4v6 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-2mvqm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-2mvqm has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lw6l6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lw6l6 has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9w425\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9w425 has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-ndzg5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-ndzg5 has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-b662v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-b662v has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-wj72x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-wj72x has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-5c5cd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-5c5cd has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lkvbz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lkvbz has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-22klt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-22klt has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-rxm4b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-rxm4b has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-r7qwh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-r7qwh has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-xhl5w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-xhl5w has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-h4p6b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-h4p6b has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-j7f54\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-j7f54 has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-bs5j7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-bs5j7 has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lwcng\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lwcng has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9n44n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9n44n has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-dp4wn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-dp4wn has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-8qlz4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-8qlz4 has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9n2td\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9n2td has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-d6x6h\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-d6x6h has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-92ktd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-92ktd has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-sxxzw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-sxxzw has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-zvv2v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-zvv2v has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-2hsfs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-2hsfs has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-nz8k8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-nz8k8 has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-gssl7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-gssl7 has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-7865d\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-7865d has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402589850472448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7338,"pr_state":"closed","pr_sha":"e49406545480a4617927868edca9ac3c5a34842c","final_merged_sha":"e49406545480a4617927868edca9ac3c5a34842c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T15:14:47Z","started_at":"2026-10-03T15:14:47Z","completed_at":"2026-10-03T17:35:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T17:44:11.0717184Z","timing_metadata_checked_at":"2026-10-03T17:44:11.0717184Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T17:44:11.075036389Z","region_metadata_checked_at":"2026-10-03T17:44:11.075036389Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"958dd735828a6f380a34b8f40e8bb7405a576cff95757bbd214a03126c2e54ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402589850472448","occurred_at":"2026-10-03T15:14:47Z","failed_at":"e2e","signature_id":"afc72a53fa7774ed1734007a58442b2c5acbb5d61e6acaf495185cd900df7590","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x1737a1124260\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kjgdvvpr4g6r\" to finish creating: GET https://rp.j0472448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"name\": \"9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T16:34:46.276618892Z\",\n \"endTime\": \"2026-10-03T16:53:53.086174722Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x1737a1124260\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kjgdvvpr4g6r\" to finish creating: GET https://rp.j0472448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"name\": \"9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T16:34:46.276618892Z\",\n \"endTime\": \"2026-10-03T16:53:53.086174722Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402405829578752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7340,"pr_state":"open","pr_sha":"e0e2b9b5f45257ab5de133a4651e3a7e85f559f0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T15:14:03Z","started_at":"2026-10-03T15:14:03Z","completed_at":"2026-10-03T17:16:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T17:23:30.858942952Z","timing_metadata_checked_at":"2026-10-03T17:23:30.858942952Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T17:23:30.865068732Z","region_metadata_checked_at":"2026-10-03T17:23:30.865068732Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0f32fc7afb0e4a5a14af2a3a7a80320a0f4e41c323870044016b5cb2b5b55af4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402405829578752","occurred_at":"2026-10-03T15:14:03Z","failed_at":"e2e","signature_id":"dca811aa20ada3f91e9119ee0eecf021dba8048b0d56a9111868dce8ce4a50a4","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x22d6ad451998\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x22d6ad451998\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"b1c110075ee77dc27f92988b75c56175ac5e389c180d06cd5a9499498d5f3510","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402405829578752","occurred_at":"2026-10-03T15:14:03Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T16:03:43Z\n Ended: 2026-10-03T16:09:42Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9578752-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106398375271206912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7326,"pr_state":"closed","pr_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","final_merged_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T14:58:02Z","started_at":"2026-10-03T14:58:02Z","completed_at":"2026-10-03T15:16:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T15:29:07.558546071Z","timing_metadata_checked_at":"2026-10-03T15:29:07.558546071Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-03T15:29:07.564085353Z","region_metadata_checked_at":"2026-10-03T16:32:06.827297578Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2c7b58542b1daa8e6ae86d29617f603a208ae41c7d97524bb6cc817bf8b8f3d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106398375271206912","occurred_at":"2026-10-03T14:58:02Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106396412521484288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7318,"pr_state":"closed","pr_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","final_merged_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T14:50:14Z","started_at":"2026-10-03T14:50:14Z","completed_at":"2026-10-03T16:56:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T17:02:50.844127768Z","timing_metadata_checked_at":"2026-10-03T17:02:50.844127768Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T17:02:50.847333458Z","region_metadata_checked_at":"2026-10-03T17:02:50.847333458Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7341/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106378931551604736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7341,"pr_state":"closed","pr_sha":"32714ac0f66cb0a9e03b0f3781d61c5812a015d5","final_merged_sha":"b870cb897e3705bbdc0b2ea937f9ccc37eee523e","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T13:40:45Z","started_at":"2026-10-03T13:40:45Z","completed_at":"2026-10-03T16:04:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T16:11:09.224238736Z","timing_metadata_checked_at":"2026-10-03T16:11:09.224238736Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T16:11:09.227116627Z","region_metadata_checked_at":"2026-10-03T16:11:09.227116627Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106365799160614912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7340,"pr_state":"open","pr_sha":"e0e2b9b5f45257ab5de133a4651e3a7e85f559f0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T12:48:35Z","started_at":"2026-10-03T12:48:35Z","completed_at":"2026-10-03T15:03:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T15:08:31.638086356Z","timing_metadata_checked_at":"2026-10-03T15:08:31.638086356Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T15:08:31.641536045Z","region_metadata_checked_at":"2026-10-03T15:08:31.641536045Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"632f2ac9558e97b5be3e0c9229021fa6a91714bd358870221b02b9886daaf2cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106365799160614912","occurred_at":"2026-10-03T12:48:35Z","failed_at":"e2e","signature_id":"b0563ce9088e2d908cb5cdc30392d78444947ed998364dd964a7182a39daa5f6","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-vzb878crbjw5\" to finish creating: GET https://rp.j0614912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"name\": \"737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T14:04:39.339033685Z\",\n \"endTime\": \"2026-10-03T14:23:40.256240312Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-vzb878crbjw5\" to finish creating: GET https://rp.j0614912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"name\": \"737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T14:04:39.339033685Z\",\n \"endTime\": \"2026-10-03T14:23:40.256240312Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106361385184137216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7318,"pr_state":"closed","pr_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","final_merged_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T12:31:03Z","started_at":"2026-10-03T12:31:03Z","completed_at":"2026-10-03T14:38:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T14:47:01.104451426Z","timing_metadata_checked_at":"2026-10-03T14:47:01.104451426Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T14:47:01.107531117Z","region_metadata_checked_at":"2026-10-03T14:47:01.107531117Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9fb13f1013352559ec9a50ccbeedb21b9bca25d368d92c390edacfc02a4f56a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106361385184137216","occurred_at":"2026-10-03T12:31:03Z","failed_at":"e2e","signature_id":"c05cf9daf6680f1855b02943b461dc08c3799199d299289700c2fc278277b2c4","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x1d83c5b77a40\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tzccwkmbcld2\" to finish creating: GET https://rp.j4137216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/63335228-135d-4644-b7d4-651deba91b70\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/63335228-135d-4644-b7d4-651deba91b70\",\n \"name\": \"63335228-135d-4644-b7d4-651deba91b70\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T13:14:48.144915783Z\",\n \"endTime\": \"2026-10-03T13:33:57.141188553Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-47kjt-tgj5p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-tzccwkmbcld2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-47kjt-tgj5p to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-47kjt-tgj5p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-tzccwkmbcld2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-47kjt-tgj5p to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x1d83c5b77a40\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tzccwkmbcld2\" to finish creating: GET https://rp.j4137216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/63335228-135d-4644-b7d4-651deba91b70\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/63335228-135d-4644-b7d4-651deba91b70\",\n \"name\": \"63335228-135d-4644-b7d4-651deba91b70\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T13:14:48.144915783Z\",\n \"endTime\": \"2026-10-03T13:33:57.141188553Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-47kjt-tgj5p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-tzccwkmbcld2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-47kjt-tgj5p to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-47kjt-tgj5p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-tzccwkmbcld2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-47kjt-tgj5p to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106355570880548864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7318,"pr_state":"closed","pr_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","final_merged_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T12:07:56Z","started_at":"2026-10-03T12:07:56Z","completed_at":"2026-10-03T12:23:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T12:28:43.776440724Z","timing_metadata_checked_at":"2026-10-03T12:28:43.776440724Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T12:28:43.779884112Z","region_metadata_checked_at":"2026-10-03T12:28:43.779884112Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"16469d0e96e4bc7f2e45e97673b95797758ae3e2f5450f6ce28accca19f79f2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106355570880548864","occurred_at":"2026-10-03T12:07:56Z","failed_at":"provision","signature_id":"beba06114dcaa7d8f5cf05070efb098c564f39c87ef41d58003f0a49b2fc4e73","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-03T12:18:04.762Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n Kind: Shell\\n Command: ./swift-vnet.sh\\n\"\ntime=2026-10-03T12:18:05.175Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Failed to load or parse file /tmp/azure-cli-config-3770628587/azureProfile.json. It will be overridden by default settings.\\nERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc","failure_pattern":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","search_query":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106344080505049088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7338,"pr_state":"closed","pr_sha":"e49406545480a4617927868edca9ac3c5a34842c","final_merged_sha":"e49406545480a4617927868edca9ac3c5a34842c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T11:22:17Z","started_at":"2026-10-03T11:22:17Z","completed_at":"2026-10-03T13:56:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T14:04:07.733045833Z","timing_metadata_checked_at":"2026-10-03T14:04:07.733045833Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T14:04:07.735609325Z","region_metadata_checked_at":"2026-10-03T14:04:07.735609325Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1dbe49f942a9b96ff4598ef6e59436df5c75556d57b50ae869741738f18d6132","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106344080505049088","occurred_at":"2026-10-03T11:22:17Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T12:40:19Z\n Ended: 2026-10-03T13:01:20Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5049088-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"384fe8b70bcf3555a05b604c55fa1a5693797acd7f5a1f8bdc1cee348a4483d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106344080505049088","occurred_at":"2026-10-03T11:22:17Z","failed_at":"e2e","signature_id":"3c3f2ccc36c35f1c9e21f03f7eff4418f1739e7896a5865697c85dc9064d117f","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d36ccbc2160\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vgn28b5mgscc\" to finish creating: GET https://rp.j5049088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1833854b-b6c4-421d-92ac-acbc850a2d4d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1833854b-b6c4-421d-92ac-acbc850a2d4d\",\n \"name\": \"1833854b-b6c4-421d-92ac-acbc850a2d4d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T13:01:15.160971572Z\",\n \"endTime\": \"2026-10-03T13:20:17.766960003Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq to exist\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq to exist\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d36ccbc2160\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vgn28b5mgscc\" to finish creating: GET https://rp.j5049088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1833854b-b6c4-421d-92ac-acbc850a2d4d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1833854b-b6c4-421d-92ac-acbc850a2d4d\",\n \"name\": \"1833854b-b6c4-421d-92ac-acbc850a2d4d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T13:01:15.160971572Z\",\n \"endTime\": \"2026-10-03T13:20:17.766960003Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq to exist\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq to exist\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c3a9ccd747392dd1e1bf958e905cac99596e3fe98e783f962a633585c7962a8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106344080505049088","occurred_at":"2026-10-03T11:22:17Z","failed_at":"e2e","signature_id":"21861e38aab43579ee30401f2af52d615d1c92a4943c93ec549c760780f28b16","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x227e9786b20\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-9q2trs\" in resourcegroup=\"rg-np-skip-minor-9q2trs-89955s58px2l\" to finish creating: GET https://rp.j5049088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c34cd-a348-440a-9cce-2255b2dba56d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c34cd-a348-440a-9cce-2255b2dba56d\",\n \"name\": \"8f9c34cd-a348-440a-9cce-2255b2dba56d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T12:59:28.477851902Z\",\n \"endTime\": \"2026-10-03T13:18:37.714240199Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9q2trs-89955s58px2l-np-sm-9q2trs--managed/providers/Microsoft.Compute/virtualMachines/e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9q2trs-89955s58px2l-np-sm-9q2trs--managed/providers/Microsoft.Compute/virtualMachines/e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x227e9786b20\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-9q2trs\" in resourcegroup=\"rg-np-skip-minor-9q2trs-89955s58px2l\" to finish creating: GET https://rp.j5049088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c34cd-a348-440a-9cce-2255b2dba56d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c34cd-a348-440a-9cce-2255b2dba56d\",\n \"name\": \"8f9c34cd-a348-440a-9cce-2255b2dba56d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T12:59:28.477851902Z\",\n \"endTime\": \"2026-10-03T13:18:37.714240199Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9q2trs-89955s58px2l-np-sm-9q2trs--managed/providers/Microsoft.Compute/virtualMachines/e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9q2trs-89955s58px2l-np-sm-9q2trs--managed/providers/Microsoft.Compute/virtualMachines/e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7334/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106339789761417216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7334,"pr_state":"closed","pr_sha":"df793e0d79069c2fd4aa1e8d0a96fc2e48b92451","final_merged_sha":"df793e0d79069c2fd4aa1e8d0a96fc2e48b92451","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T11:05:14Z","started_at":"2026-10-03T11:05:14Z","completed_at":"2026-10-03T13:54:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T14:04:07.554863801Z","timing_metadata_checked_at":"2026-10-03T14:04:07.554863801Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T14:04:07.55831059Z","region_metadata_checked_at":"2026-10-03T14:04:07.55831059Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106319269334618112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7318,"pr_state":"closed","pr_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","final_merged_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T09:43:41Z","started_at":"2026-10-03T09:43:41Z","completed_at":"2026-10-03T11:59:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T12:07:43.864622856Z","timing_metadata_checked_at":"2026-10-03T12:07:43.864622856Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T12:07:43.868061041Z","region_metadata_checked_at":"2026-10-03T12:07:43.868061041Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"cf72fcc6a8a8329c7bf827e3b98a700cdf365d68eafb72a87cfd8524f6be6876","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106319269334618112","occurred_at":"2026-10-03T09:43:41Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T10:35:51Z\n Ended: 2026-10-03T10:41:49Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4618112-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7268/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106316215747088384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7268,"pr_state":"open","pr_sha":"e83df41de9079bdb5b597c55cd9681d618094984","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T09:31:33Z","started_at":"2026-10-03T09:31:33Z","completed_at":"2026-10-03T11:56:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T12:07:43.46503529Z","timing_metadata_checked_at":"2026-10-03T12:07:43.46503529Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T12:07:43.468611574Z","region_metadata_checked_at":"2026-10-03T12:07:43.468611574Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fee8aa102f2d89b6f24927d984b018cb78bd5f62ef88f2013fb5ed59bcea442d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7268/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106316215747088384","occurred_at":"2026-10-03T09:31:33Z","failed_at":"e2e","signature_id":"8e852f466cf78bfd32f7a3098547e4a2279ba62592d4769dc239478e539ec985","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ccede026220\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-74v4krpx78dr\" to finish creating: GET https://rp.j7088384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61d03357-4fb7-49b2-a777-a99ffa662551\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61d03357-4fb7-49b2-a777-a99ffa662551\",\n \"name\": \"61d03357-4fb7-49b2-a777-a99ffa662551\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T10:52:21.517435656Z\",\n \"endTime\": \"2026-10-03T11:11:28.16247644Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st to exist\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st to exist\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ccede026220\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-74v4krpx78dr\" to finish creating: GET https://rp.j7088384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61d03357-4fb7-49b2-a777-a99ffa662551\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61d03357-4fb7-49b2-a777-a99ffa662551\",\n \"name\": \"61d03357-4fb7-49b2-a777-a99ffa662551\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T10:52:21.517435656Z\",\n \"endTime\": \"2026-10-03T11:11:28.16247644Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st to exist\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st to exist\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106307424179720192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"closed","pr_sha":"1495f07f761e3c094aa60549d95c4cf2cf1748e9","final_merged_sha":"0d147555918839b4a556ae5981b12356ac4cf337","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T08:56:37Z","started_at":"2026-10-03T08:56:37Z","completed_at":"2026-10-03T11:38:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T11:46:31.91349312Z","timing_metadata_checked_at":"2026-10-03T11:46:31.91349312Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T11:46:31.91656301Z","region_metadata_checked_at":"2026-10-03T11:46:31.91656301Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7334/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106297012751175680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7334,"pr_state":"closed","pr_sha":"df793e0d79069c2fd4aa1e8d0a96fc2e48b92451","final_merged_sha":"df793e0d79069c2fd4aa1e8d0a96fc2e48b92451","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T08:15:15Z","started_at":"2026-10-03T08:15:15Z","completed_at":"2026-10-03T10:29:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T10:32:36.968058061Z","timing_metadata_checked_at":"2026-10-03T10:32:36.968058061Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T10:32:36.971898148Z","region_metadata_checked_at":"2026-10-03T10:32:36.971898148Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"e42b1436444203f19d22144aa8b53e759589613daaa489597343618125640705","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7334/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106297012751175680","occurred_at":"2026-10-03T08:15:15Z","failed_at":"alert","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T09:10:27Z\n Ended: 2026-10-03T09:28:26Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1175680-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T09:15:27Z\n Ended: 2026-10-03T09:28:26Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1175680-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106283184227356672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7318,"pr_state":"closed","pr_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","final_merged_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T07:20:18Z","started_at":"2026-10-03T07:20:18Z","completed_at":"2026-10-03T09:38:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T09:50:21.000624046Z","timing_metadata_checked_at":"2026-10-03T09:50:21.000624046Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T09:50:21.003638035Z","region_metadata_checked_at":"2026-10-03T09:50:21.003638035Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"8a1bc5b64cfc6a39fac7805c805c1e051d5f1c84b09024f43f0817ee0aeb74b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106283184227356672","occurred_at":"2026-10-03T07:20:18Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T08:23:18Z\n Ended: 2026-10-03T08:30:15Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7356672-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232246682783744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7336,"pr_state":"closed","pr_sha":"0c82ea710333dd869bbd1a24686f11e37cfe9b99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T03:57:54Z","started_at":"2026-10-03T03:57:54Z","completed_at":"2026-10-03T06:21:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T06:42:21.214113117Z","timing_metadata_checked_at":"2026-10-03T06:42:21.214113117Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T06:42:21.217856805Z","region_metadata_checked_at":"2026-10-03T06:42:21.217856805Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"10fed181611039bb684ee91aa7d3fa3ec49d7fd995949d165d2d931b8a9493d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232246682783744","occurred_at":"2026-10-03T03:57:54Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T05:06:36Z\n Ended: 2026-10-03T05:12:36Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2783744-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"dbf50194192ece1b1a6409cf6bdaea3b43fe3b2554d4508bd2440e79909d3914","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232246682783744","occurred_at":"2026-10-03T03:57:54Z","failed_at":"e2e","signature_id":"4ecba326214d91b223bb8a28fa503efb6110b5f309f19014e5eec742eee54784","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x1229c1294580\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-q4mxjczq7tsd\" to finish creating: GET https://rp.j2783744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7757c823-88ba-458e-aed3-9dfdac2d7080\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7757c823-88ba-458e-aed3-9dfdac2d7080\",\n \"name\": \"7757c823-88ba-458e-aed3-9dfdac2d7080\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T05:03:30.952777318Z\",\n \"endTime\": \"2026-10-03T05:22:34.856876402Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-62mv4-kp8nq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-q4mxjczq7tsd--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-62mv4-kp8nq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-62mv4-kp8nq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-q4mxjczq7tsd--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-62mv4-kp8nq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x1229c1294580\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-q4mxjczq7tsd\" to finish creating: GET https://rp.j2783744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7757c823-88ba-458e-aed3-9dfdac2d7080\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7757c823-88ba-458e-aed3-9dfdac2d7080\",\n \"name\": \"7757c823-88ba-458e-aed3-9dfdac2d7080\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T05:03:30.952777318Z\",\n \"endTime\": \"2026-10-03T05:22:34.856876402Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-62mv4-kp8nq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-q4mxjczq7tsd--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-62mv4-kp8nq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-62mv4-kp8nq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-q4mxjczq7tsd--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-62mv4-kp8nq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232206048366592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7335,"pr_state":"closed","pr_sha":"b25f2cbc29ff36f009c24cff5cd46c29b5a264ab","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T03:57:44Z","started_at":"2026-10-03T03:57:44Z","completed_at":"2026-10-03T06:19:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T06:31:30.403522951Z","timing_metadata_checked_at":"2026-10-03T06:31:30.403522951Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T06:31:30.40698534Z","region_metadata_checked_at":"2026-10-03T06:31:30.40698534Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b08aefbb1959664a9df2d569690d04010f1efa0b569d2db1644f86f9768ee6d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232206048366592","occurred_at":"2026-10-03T03:57:44Z","failed_at":"e2e","signature_id":"4cb0132308929f56cf10c7cf8dcc82eb33338d500ae0ff69d7f2b06b20772f9b","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x331a5ee39be0\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-hzfdbcnlckj2\" to finish creating: GET https://rp.j8366592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7eb0be6-172b-405f-8019-aa4a01912e9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7eb0be6-172b-405f-8019-aa4a01912e9a\",\n \"name\": \"f7eb0be6-172b-405f-8019-aa4a01912e9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T05:21:56.267331547Z\",\n \"endTime\": \"2026-10-03T05:41:00.127542846Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-hzfdbcnlckj2--managed/providers/Microsoft.Compute/virtualMachines/f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-hzfdbcnlckj2--managed/providers/Microsoft.Compute/virtualMachines/f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x331a5ee39be0\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-hzfdbcnlckj2\" to finish creating: GET https://rp.j8366592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7eb0be6-172b-405f-8019-aa4a01912e9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7eb0be6-172b-405f-8019-aa4a01912e9a\",\n \"name\": \"f7eb0be6-172b-405f-8019-aa4a01912e9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T05:21:56.267331547Z\",\n \"endTime\": \"2026-10-03T05:41:00.127542846Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-hzfdbcnlckj2--managed/providers/Microsoft.Compute/virtualMachines/f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-hzfdbcnlckj2--managed/providers/Microsoft.Compute/virtualMachines/f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106223756383883264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7332,"pr_state":"open","pr_sha":"3fd6df7d8888145d8258955876aeb97dfd0efbb8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T03:24:09Z","started_at":"2026-10-03T03:24:09Z","completed_at":"2026-10-03T05:51:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T06:10:10.214122465Z","timing_metadata_checked_at":"2026-10-03T06:10:10.214122465Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T06:10:10.218235452Z","region_metadata_checked_at":"2026-10-03T06:10:10.218235452Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"152f667948b6ca5fd63b8ccff02f563599d770f85f69db6a6541aebf8c261bee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106223756383883264","occurred_at":"2026-10-03T03:24:09Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T04:38:45Z\n Ended: 2026-10-03T04:44:46Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3883264-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"6fccb2861b1ecbef1d1d2954b420844ad3d2eda4e3ea3e72b320c03b8674b140","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106223756383883264","occurred_at":"2026-10-03T03:24:09Z","failed_at":"alert","signature_id":"fee414b15571075bb38e6218ae18960a82bae3ac8dd2273bf21bea06f37d99a3","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-03T04:27:33Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j3883264-svc\", component=\"backend\", name=\"openshiftversioncatalog\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue openshiftversioncatalog has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"c44920fe390496e7de07f1b46f2a57881e5bb54239316b8d36e46c7b09ad07f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106223756383883264","occurred_at":"2026-10-03T03:24:09Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106216516587360256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7333,"pr_state":"closed","pr_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","final_merged_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T02:55:23Z","started_at":"2026-10-03T02:55:23Z","completed_at":"2026-10-03T05:20:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T05:38:36.895533037Z","timing_metadata_checked_at":"2026-10-03T05:38:36.895533037Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T05:38:36.899385524Z","region_metadata_checked_at":"2026-10-03T05:38:36.899385524Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5bbb640259ead5e9f3e0b0d6fc954c806d2bde688e66f62cafc276c85df15087","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106216516587360256","occurred_at":"2026-10-03T02:55:23Z","failed_at":"e2e","signature_id":"405132e552ca41557cf8c6f086fdcc0c25a7a484630f63163bb4bed7e7d238ad","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0x1113bcb9c160\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-rckbfw\" in resourcegroup=\"rg-np-downgrade-rckbfw-856zsqvm5rkb\" to finish creating: GET https://rp.j7360256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/074aacd3-7322-43b0-9305-f06220347ff4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/074aacd3-7322-43b0-9305-f06220347ff4\",\n \"name\": \"074aacd3-7322-43b0-9305-f06220347ff4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T04:27:23.785116321Z\",\n \"endTime\": \"2026-10-03T04:46:25.26888513Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-rckbfw-856zsqvm5rkb-np-dg-rckbfw--managed/providers/Microsoft.Compute/virtualMachines/l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-rckbfw-856zsqvm5rkb-np-dg-rckbfw--managed/providers/Microsoft.Compute/virtualMachines/l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0x1113bcb9c160\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-rckbfw\" in resourcegroup=\"rg-np-downgrade-rckbfw-856zsqvm5rkb\" to finish creating: GET https://rp.j7360256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/074aacd3-7322-43b0-9305-f06220347ff4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/074aacd3-7322-43b0-9305-f06220347ff4\",\n \"name\": \"074aacd3-7322-43b0-9305-f06220347ff4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T04:27:23.785116321Z\",\n \"endTime\": \"2026-10-03T04:46:25.26888513Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-rckbfw-856zsqvm5rkb-np-dg-rckbfw--managed/providers/Microsoft.Compute/virtualMachines/l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-rckbfw-856zsqvm5rkb-np-dg-rckbfw--managed/providers/Microsoft.Compute/virtualMachines/l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cd4eb016a6af8364f67921f741d6d11977f17449764ccf3e8e4e9a9cbbb080e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106216516587360256","occurred_at":"2026-10-03T02:55:23Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T04:07:22Z\n Ended: 2026-10-03T04:13:25Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7360256-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106201793171034112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7331,"pr_state":"closed","pr_sha":"53a4ed21d7b546b4f4ac2fb406d7d9bec086289b","final_merged_sha":"d5ee24cc09296f5a16b33a9dbe4bd808f7e62ec2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T01:56:53Z","started_at":"2026-10-03T01:56:53Z","completed_at":"2026-10-03T04:25:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T04:35:41.037657768Z","timing_metadata_checked_at":"2026-10-03T04:35:41.037657768Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T04:35:41.042354053Z","region_metadata_checked_at":"2026-10-03T04:35:41.042354053Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"977a2c03fc29820644e6afcb40430d4dd99b5781670499f0c4ab041c051a2379","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106201793171034112","occurred_at":"2026-10-03T01:56:53Z","failed_at":"alert","signature_id":"fa412034903709961f37b7b9179b2905b4c4fb3d1b051e1833d51c25494e7359","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T03:07:49Z\n Ended: 2026-10-03T03:19:50Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1034112-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T03:08:45Z\n Ended: 2026-10-03T03:22:45Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1034112-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106186045094629376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7335,"pr_state":"closed","pr_sha":"b25f2cbc29ff36f009c24cff5cd46c29b5a264ab","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T00:54:18Z","started_at":"2026-10-03T00:54:18Z","completed_at":"2026-10-03T03:16:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T03:32:21.511611898Z","timing_metadata_checked_at":"2026-10-03T03:32:21.511611898Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T03:32:21.515623085Z","region_metadata_checked_at":"2026-10-03T03:32:21.515623085Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"07695946dec4d21ca6bb36750251bb90756265e3fa3ee37e4dae4e6486c1e5de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106186045094629376","occurred_at":"2026-10-03T00:54:18Z","failed_at":"alert","signature_id":"fa412034903709961f37b7b9179b2905b4c4fb3d1b051e1833d51c25494e7359","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T01:56:32Z\n Ended: 2026-10-03T02:03:33Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4629376-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T01:57:33Z\n Ended: 2026-10-03T02:10:32Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4629376-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"3b3e7b53ab568a28c1b36ad2580873415bbd2c69f08b74aca967f8a4556e68b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106186045094629376","occurred_at":"2026-10-03T00:54:18Z","failed_at":"e2e","signature_id":"5bfc63de429111a26e0e3c0120b6821322754898f203dabd0929d69d6732c4f1","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3263781c03a0\u003e: \n DNS for route host agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3263781c03a0\u003e: \n DNS for route host agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106184864796512256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7336,"pr_state":"closed","pr_sha":"0c82ea710333dd869bbd1a24686f11e37cfe9b99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T00:49:37Z","started_at":"2026-10-03T00:49:37Z","completed_at":"2026-10-03T03:19:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T03:32:21.69026701Z","timing_metadata_checked_at":"2026-10-03T03:32:21.69026701Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T03:32:21.692941801Z","region_metadata_checked_at":"2026-10-03T03:32:21.692941801Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0dba5f7c0186f27a7716c513c7f465b1f6ca6155f811f6c3c341b55d7094071d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106184864796512256","occurred_at":"2026-10-03T00:49:37Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"0f550217af47207c5f9de35b57e8c8dc3b2e55e73691648da225c96058fa2c4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106184864796512256","occurred_at":"2026-10-03T00:49:37Z","failed_at":"e2e","signature_id":"12b342095018d1a5c43a8d0f0915ddf5043eaa8b66110e00ae45d0793de2806b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-xt2c7s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b2c75ad8a80\u003e: \n DNS for route host agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-xt2c7s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b2c75ad8a80\u003e: \n DNS for route host agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"df977953ff39d5d045e2981e35f026ff36bc89407bfbb1f3db17e4f038cdc89e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106184864796512256","occurred_at":"2026-10-03T00:49:37Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T02:01:45Z\n Ended: 2026-10-03T02:07:46Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6512256-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106183789494407168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7333,"pr_state":"closed","pr_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","final_merged_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T00:45:21Z","started_at":"2026-10-03T00:45:21Z","completed_at":"2026-10-03T02:03:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T02:18:30.352206704Z","timing_metadata_checked_at":"2026-10-03T02:18:30.352206704Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T02:18:30.355040096Z","region_metadata_checked_at":"2026-10-03T02:18:30.355040096Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"5b98b67a95c3198431956740459473f5d79d9c63aabfee380becbad2411b3aa8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106183789494407168","occurred_at":"2026-10-03T00:45:21Z","failed_at":"provision","signature_id":"f292884abfee87bf70d297d035f626393e7b29fe680eeec4ec5586d3b8b28c52","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"templatize-pipeline","failure_text":"time=2026-10-03T01:35:28.208Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-03T01:35:28.778Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy values=\"map[azureArmHelperIdentityCertName:armHelperPoolCert-10 azureArmHelperIdentityClientId:0a9e3940-6bc3-4cbf-a8f7-c65c3d8d2462 azureArmHelperMockFpaPrincipalId:47f69502-0065-4d9a-b19b-d403e183d2f4 azureCloudEnvironmentName:AzurePublicCloud azureCsMiClientId:de4f7be2-47e3-4002-adc2-77c2fcdec830 azureFirstPartyApplicationClientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX azureMiMockServicePrincipalCertName:msiMockPoolCert-17 azureMiMockServicePrincipalClientId:e7112f7f-5b45-480e-bc44-abf9f219e6f0 azureMiMockServicePrincipalPrincipalId:39eefc4f-5e24-491d-9f09-bcc0ad573bcf azureOperatorsMI:map[dev:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] public:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] roleSetName:dev] backplaneURL: batchProcesses: batchProcessesDryRun:true clientId:foo clientScopes:openid clientSecret:bar clusterErrorReport:false cosmosDatabaseName:arohcpci01-rp-j4407168 cosmosDatabaseUrl:https://arohcpci01-rp-j4407168.documents.azure.com:443/ cpuRequest:630m csDeploymentStrategy:map[rollingUpdate:map[maxSurge:1 maxUnavailable:0] type:RollingUpdate] dataPlaneHAProxyImage:arohcpocpdev.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678 databaseAuthMethod:postgres databaseDisableTls:true databaseHost:ocm-cs-db databaseK8sSecretName:ocm-cs-db databaseName:ocm-cs-db databasePassword:TheBlurstOfTimes databasePort:5432 databaseServiceName:ocm-cs-db databaseUser:ocm databaseVolumeCapacity:512Mi debugMaxDuration:5m debugPort: defaultExpiration:0 deployment:map[zoneCount:3] environment:arohcpci01 forceMigration: fpaCertName:firstPartyCert2 gatewayUrl:http://127.0.0.1:9090 global:map[imageDigest:sha256:5fffce295f8b49be38ae2385714d09af29700dd3dc60c40a058f7aeb66e97d72] imageDigest:sha256:5fffce295f8b49be38ae2385714d09af29700dd3dc60c40a058f7aeb66e97d72 imageRegistry:arohcpsvcdev.azurecr.io imageRepository:app-sre/aro-hcp-clusters-service insecure:false jwksUrl:http://localhost logLevel:4 machinePoolMigrationWorkerPeriod:1h managedIdentitiesDataPlaneAudienceResource:https://dummy.org maximumExpiration:0 memoryLimit:unlimited memoryRequest:150Mi ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev ocpAcrResourceUrl:arohcpocpdev.azurecr.io oidcIssuerBaseUrl:https://arohcpci01oidcj4407168.z9.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj4407168.blob.core.windows.net/ postgresqlVersion:12 provisionShardClusterLimit:100 pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:app-sre/aro-hcp-clusters-service:pull workloadIdentityClientId:27f533be-4c23-4032-9d19-da9c7454bd3b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] region:canadacentral replicas:3 runtimeMode:aro-hcp serviceAccountName:clusters-service serviceKeyVaultName:aro-hcp-dev-svc-kv tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesEnabled:true tlsCertificatesIssuer:Self tokenUrl:http://localhost useAzureDB:false userDefinedDnsBaseDomain:i1.devshift.org]\"\ntime=2026-10-03T01:35:28.779Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-03T01:35:29.089Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy namespace=clusters-service\ntime=2026-10-03T01:35:29.415Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy dryRun=false\ntime=2026-10-03T01:35:29.416Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-03T01:35:29.465Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-03T01:45:35.236Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-10-03T01:45:35.237Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a8bb4542e08df91dbf01364e592eda2b949d63ea36dbf43a83cbdbd4c51b7dfe","failure_pattern":"failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded"}},{"row_id":"6666431c752f574bfbdf44722e36ce746442fea8bf646ae33fad2cd76e67b21f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106183789494407168","occurred_at":"2026-10-03T00:45:21Z","failed_at":"alert","signature_id":"5772227d0b1c3dbc1f88ff19aefa92151e392000e5bb31a2f32a22c36981ef0c","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-03T01:40:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4407168-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-74695d59f9-hxdpn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-74695d59f9-hxdpn has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-03T01:42:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4407168-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-74695d59f9-bfnvs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-74695d59f9-bfnvs has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-03T01:42:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4407168-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-74695d59f9-wjv8w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-74695d59f9-wjv8w has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","a8bb4542e08df91dbf01364e592eda2b949d63ea36dbf43a83cbdbd4c51b7dfe"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}