{"meta":{"date":"2026-10-03","timezone":"UTC","generated_at":"2026-10-14T11:00:36Z","environments":["dev","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":39,"failed_runs":27,"runs_with_occurrences":27,"runs_with_failure_pattern_matches":26,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":12,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106522529362350080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7332,"pr_state":"open","pr_sha":"9d3908a862dc12dea2ae6969184167ee3f53c194","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T23:11:22Z","started_at":"2026-10-03T23:11:22Z","completed_at":"2026-10-04T01:44:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-04T01:51:07.673019618Z","timing_metadata_checked_at":"2026-10-04T01:51:07.673019618Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-04T01:51:07.680543294Z","region_metadata_checked_at":"2026-10-04T01:51:07.680543294Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"07154007428468f20f1112009c5fb6c98549cc6d2bceda73a68d9d99e31f329e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106522529362350080","occurred_at":"2026-10-03T23:11:22Z","failed_at":"e2e","signature_id":"501495b0aef70c6325fa26b9f0b745d7b4ce463804d55b753046dc2f9f5ed6e4","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f72d213c320\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-n6bsrg\" in resourcegroup=\"rg-np-noedge-n6bsrg-vxj5p9qrl42z\" to finish creating: GET https://rp.j2350080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\",\n \"name\": \"83d90b9f-3285-4912-9884-40a78690862b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-04T00:37:10.755457782Z\",\n \"endTime\": \"2026-10-04T00:56:17.299103912Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f72d213c320\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-n6bsrg\" in resourcegroup=\"rg-np-noedge-n6bsrg-vxj5p9qrl42z\" to finish creating: GET https://rp.j2350080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\",\n \"name\": \"83d90b9f-3285-4912-9884-40a78690862b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-04T00:37:10.755457782Z\",\n \"endTime\": \"2026-10-04T00:56:17.299103912Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106519157846577152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7333,"pr_state":"closed","pr_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","final_merged_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T22:57:59Z","started_at":"2026-10-03T22:57:59Z","completed_at":"2026-10-04T01:26:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-04T01:29:47.036933122Z","timing_metadata_checked_at":"2026-10-04T01:29:47.036933122Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-04T01:29:47.040246812Z","region_metadata_checked_at":"2026-10-04T01:29:47.040246812Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106498996229902336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7338,"pr_state":"closed","pr_sha":"e49406545480a4617927868edca9ac3c5a34842c","final_merged_sha":"e49406545480a4617927868edca9ac3c5a34842c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T21:37:52Z","started_at":"2026-10-03T21:37:52Z","completed_at":"2026-10-04T00:08:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-04T00:15:44.473924569Z","timing_metadata_checked_at":"2026-10-04T00:15:44.473924569Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-04T00:15:44.47677176Z","region_metadata_checked_at":"2026-10-04T00:15:44.47677176Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106489060980690944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7336,"pr_state":"closed","pr_sha":"0c82ea710333dd869bbd1a24686f11e37cfe9b99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T20:58:23Z","started_at":"2026-10-03T20:58:23Z","completed_at":"2026-10-03T23:26:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T23:44:22.460109596Z","timing_metadata_checked_at":"2026-10-03T23:44:22.460109596Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T23:44:22.521655699Z","region_metadata_checked_at":"2026-10-03T23:44:22.521655699Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"1cffcfd8e63a16fab06bc543d2dddcc4d75a7d0783b479870d6a4a474731e2cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106489060980690944","occurred_at":"2026-10-03T20:58:23Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T22:09:35Z\n Ended: 2026-10-03T22:15:33Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0690944-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7341/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106478441103626240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7341,"pr_state":"closed","pr_sha":"b870cb897e3705bbdc0b2ea937f9ccc37eee523e","final_merged_sha":"b870cb897e3705bbdc0b2ea937f9ccc37eee523e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T20:15:49Z","started_at":"2026-10-03T20:15:49Z","completed_at":"2026-10-03T22:37:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T22:52:24.373501246Z","timing_metadata_checked_at":"2026-10-03T22:52:24.373501246Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T22:52:24.376685635Z","region_metadata_checked_at":"2026-10-03T22:52:24.376685635Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106473224001818624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7331,"pr_state":"closed","pr_sha":"53a4ed21d7b546b4f4ac2fb406d7d9bec086289b","final_merged_sha":"d5ee24cc09296f5a16b33a9dbe4bd808f7e62ec2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T19:55:27Z","started_at":"2026-10-03T19:55:27Z","completed_at":"2026-10-03T22:23:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T22:31:19.620134079Z","timing_metadata_checked_at":"2026-10-03T22:31:19.620134079Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T22:31:19.639654719Z","region_metadata_checked_at":"2026-10-03T22:31:19.639654719Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"7ab559cff011f1b66c12ed891d2484e47a330df2be2259018021d4f21d3aef1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106473224001818624","occurred_at":"2026-10-03T19:55:27Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T21:07:34Z\n Ended: 2026-10-03T21:13:37Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1818624-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106459631151747072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7332,"pr_state":"open","pr_sha":"8ed2be90842084d1da211b08e7ba64d1a9ba3c4a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T19:01:06Z","started_at":"2026-10-03T19:01:06Z","completed_at":"2026-10-03T21:29:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T21:35:17.526106349Z","timing_metadata_checked_at":"2026-10-03T21:35:17.526106349Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T21:35:17.529442839Z","region_metadata_checked_at":"2026-10-03T21:35:17.529442839Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106458234737922048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7333,"pr_state":"closed","pr_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","final_merged_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T18:55:53Z","started_at":"2026-10-03T18:55:53Z","completed_at":"2026-10-03T21:30:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T21:35:17.745416578Z","timing_metadata_checked_at":"2026-10-03T21:35:17.745416578Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T21:35:17.748356869Z","region_metadata_checked_at":"2026-10-03T21:35:17.748356869Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e92de00c355c6d1e81532aaf6a4b73f81fd2296ad8bab01ff3068380ee421fc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106458234737922048","occurred_at":"2026-10-03T18:55:53Z","failed_at":"e2e","signature_id":"79b90ae5312ada738ac4440b4501fdd97f482ae1f84f6796798dab482e44cbd8","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3528a4bdeac0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vljzr5rw54g9\" to finish creating: GET https://rp.j7922048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"name\": \"336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T20:19:49.133571808Z\",\n \"endTime\": \"2026-10-03T20:38:58.66181942Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 3 of 3 machines are not healthy\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 3 of 3 machines are not ready\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3528a4bdeac0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vljzr5rw54g9\" to finish creating: GET https://rp.j7922048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"name\": \"336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T20:19:49.133571808Z\",\n \"endTime\": \"2026-10-03T20:38:58.66181942Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 3 of 3 machines are not healthy\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 3 of 3 machines are not ready\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106455133536653312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7340,"pr_state":"open","pr_sha":"e0e2b9b5f45257ab5de133a4651e3a7e85f559f0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T18:43:34Z","started_at":"2026-10-03T18:43:34Z","completed_at":"2026-10-03T21:20:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T21:25:11.045254184Z","timing_metadata_checked_at":"2026-10-03T21:25:11.045254184Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T21:25:11.049137372Z","region_metadata_checked_at":"2026-10-03T21:25:11.049137372Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106455066687836160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7338,"pr_state":"closed","pr_sha":"e49406545480a4617927868edca9ac3c5a34842c","final_merged_sha":"e49406545480a4617927868edca9ac3c5a34842c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T18:43:18Z","started_at":"2026-10-03T18:43:18Z","completed_at":"2026-10-03T21:06:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T21:14:13.230244916Z","timing_metadata_checked_at":"2026-10-03T21:14:13.230244916Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T21:14:13.233906904Z","region_metadata_checked_at":"2026-10-03T21:14:13.233906904Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a97fadc90aeb65076d289bf4b43a4b8b8ca1b1766a94e971dbd614fba091911b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106455066687836160","occurred_at":"2026-10-03T18:43:18Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T19:53:25Z\n Ended: 2026-10-03T19:59:26Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7836160-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106451370914615296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7325,"pr_state":"closed","pr_sha":"db96e7598ff60f6d48668a2a08400759438dbc38","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T18:28:37Z","started_at":"2026-10-03T18:28:37Z","completed_at":"2026-10-03T20:57:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T21:03:35.174982068Z","timing_metadata_checked_at":"2026-10-03T21:03:35.174982068Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T21:03:35.18103285Z","region_metadata_checked_at":"2026-10-03T21:03:35.18103285Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428533512343552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7326,"pr_state":"closed","pr_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","final_merged_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T16:57:52Z","started_at":"2026-10-03T16:57:52Z","completed_at":"2026-10-03T19:47:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T20:01:11.353963696Z","timing_metadata_checked_at":"2026-10-03T20:01:11.353963696Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T20:01:11.356680087Z","region_metadata_checked_at":"2026-10-03T20:01:11.356680087Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428486020239360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7336,"pr_state":"closed","pr_sha":"0c82ea710333dd869bbd1a24686f11e37cfe9b99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T16:57:41Z","started_at":"2026-10-03T16:57:41Z","completed_at":"2026-10-03T19:45:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T19:51:09.360703987Z","timing_metadata_checked_at":"2026-10-03T19:51:09.360703987Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T19:51:09.363181579Z","region_metadata_checked_at":"2026-10-03T19:51:09.363181579Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"57308dc196e9b749fed5f9c99022a857683b2f3b413fdc447d1728eb8495a540","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428486020239360","occurred_at":"2026-10-03T16:57:41Z","failed_at":"e2e","signature_id":"09d3516d5937c201911e513dc777964c08f8d8e605d4d938d29eb1e35d21ff64","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1590b4a34e00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-hd5zgd7fw6ql\" to finish creating: GET https://rp.j0239360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"name\": \"0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T18:22:53.322060806Z\",\n \"endTime\": \"2026-10-03T18:41:55.898243278Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkgjk-bhjgr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkgjk-bhjgr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1590b4a34e00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-hd5zgd7fw6ql\" to finish creating: GET https://rp.j0239360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"name\": \"0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T18:22:53.322060806Z\",\n \"endTime\": \"2026-10-03T18:41:55.898243278Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkgjk-bhjgr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkgjk-bhjgr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428469284966400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7335,"pr_state":"closed","pr_sha":"b25f2cbc29ff36f009c24cff5cd46c29b5a264ab","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T16:57:37Z","started_at":"2026-10-03T16:57:37Z","completed_at":"2026-10-03T19:37:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T19:51:09.12188513Z","timing_metadata_checked_at":"2026-10-03T19:51:09.12188513Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T19:51:09.124214823Z","region_metadata_checked_at":"2026-10-03T19:51:09.124214823Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428464000143360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7331,"pr_state":"closed","pr_sha":"53a4ed21d7b546b4f4ac2fb406d7d9bec086289b","final_merged_sha":"d5ee24cc09296f5a16b33a9dbe4bd808f7e62ec2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T16:57:35Z","started_at":"2026-10-03T16:57:35Z","completed_at":"2026-10-03T19:40:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T19:51:09.028565921Z","timing_metadata_checked_at":"2026-10-03T19:51:09.028565921Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T19:51:09.031641011Z","region_metadata_checked_at":"2026-10-03T19:51:09.031641011Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"b2d1210b2af6cff0965edc302a3f5d3cf983f0060e1eae9ab8f9bdca8743f586","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428464000143360","occurred_at":"2026-10-03T16:57:35Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T18:23:19Z\n Ended: 2026-10-03T18:38:18Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0143360-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"d894c32d0218011a1977b9caf03d11f629e802277e3530ff3c23bca403be3fde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428464000143360","occurred_at":"2026-10-03T16:57:35Z","failed_at":"alert","signature_id":"fbbc2741d1268c4182e25737f2913295ff9bf11b2acf044ba433a4c9be14b821","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-file-csi-driver-operator-56bd98ff47-n9mw7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-file-csi-driver-operator-56bd98ff47-n9mw7 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-disk-csi-driver-operator-68446dbd4d-gp5r7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-disk-csi-driver-operator-68446dbd4d-gp5r7 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-disk-csi-driver-operator-68446dbd4d-72x46\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-disk-csi-driver-operator-68446dbd4d-72x46 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-disk-csi-driver-operator-68446dbd4d-l8w9v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-disk-csi-driver-operator-68446dbd4d-l8w9v has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-file-csi-driver-operator-56bd98ff47-lhdbd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-file-csi-driver-operator-56bd98ff47-lhdbd has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106422169931091968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7333,"pr_state":"closed","pr_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","final_merged_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T16:32:35Z","started_at":"2026-10-03T16:32:35Z","completed_at":"2026-10-03T18:47:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T18:59:24.668972577Z","timing_metadata_checked_at":"2026-10-03T18:59:24.668972577Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T18:59:24.672386065Z","region_metadata_checked_at":"2026-10-03T18:59:24.672386065Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"95ba3558ad9d83f64fca97491bc033a5a7b9f82bc6e5448545703a90d8923568","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106422169931091968","occurred_at":"2026-10-03T16:32:35Z","failed_at":"e2e","signature_id":"a7a5e0e36a34359f8084193c2d0faab4839b075dfe3e59cc885f6fb7a3743b3d","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xd9700701640\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l9xkxj2bpszs\" to finish creating: GET https://rp.j1091968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"name\": \"d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T17:56:20.630751478Z\",\n \"endTime\": \"2026-10-03T18:15:20.975618436Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xd9700701640\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l9xkxj2bpszs\" to finish creating: GET https://rp.j1091968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"name\": \"d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T17:56:20.630751478Z\",\n \"endTime\": \"2026-10-03T18:15:20.975618436Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l9xkxj2bpszs\" to finish creating: GET https://rp"}},{"row_id":"cdfd22a749c219a348e8d80d869b67c403a54500cebd607e6fd6668db12ff6b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106422169931091968","occurred_at":"2026-10-03T16:32:35Z","failed_at":"alert","signature_id":"d03789470daaa6e20f0e10af6ab181b291bf54e6bdce46582adbae8892beb283","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 69 time(s) (63 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-52l2j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-52l2j has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-vdz8t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-vdz8t has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-k2787\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-k2787 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-zp9nc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-zp9nc has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-vk8sg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-vk8sg has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-fvs76\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-fvs76 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-llstd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-llstd has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-5mcng\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-5mcng has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9sqdg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9sqdg has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:51:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-btfk4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-btfk4 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-dq2vz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-dq2vz has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-x6wz7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-x6wz7 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-ctpt9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-ctpt9 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-j2d9z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-j2d9z has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-4h79j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-4h79j has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-z2w4b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-z2w4b has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-nkt4w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-nkt4w has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-ffhbr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-ffhbr has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-csvsv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-csvsv has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-s4h6v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-s4h6v has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-dl5l4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-dl5l4 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-47862\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-47862 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-gw8rm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-gw8rm has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-mrcs5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-mrcs5 has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-tzs9x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-tzs9x has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-pjk6c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-pjk6c has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-vdtmc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-vdtmc has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-7kmn4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-7kmn4 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-6w4zc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-6w4zc has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-mnfxd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-mnfxd has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-m84qg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-m84qg has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-j8kmn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-j8kmn has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-r827x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-r827x has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lscgk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lscgk has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-gj4v6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-gj4v6 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-2mvqm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-2mvqm has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lw6l6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lw6l6 has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9w425\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9w425 has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-ndzg5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-ndzg5 has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-b662v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-b662v has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-wj72x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-wj72x has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-5c5cd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-5c5cd has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lkvbz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lkvbz has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-22klt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-22klt has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-rxm4b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-rxm4b has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-r7qwh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-r7qwh has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-xhl5w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-xhl5w has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-h4p6b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-h4p6b has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-j7f54\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-j7f54 has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-bs5j7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-bs5j7 has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lwcng\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lwcng has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9n44n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9n44n has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-dp4wn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-dp4wn has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-8qlz4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-8qlz4 has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9n2td\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9n2td has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-d6x6h\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-d6x6h has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-92ktd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-92ktd has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-sxxzw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-sxxzw has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-zvv2v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-zvv2v has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-2hsfs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-2hsfs has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-nz8k8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-nz8k8 has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-gssl7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-gssl7 has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-7865d\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-7865d has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402589850472448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7338,"pr_state":"closed","pr_sha":"e49406545480a4617927868edca9ac3c5a34842c","final_merged_sha":"e49406545480a4617927868edca9ac3c5a34842c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T15:14:47Z","started_at":"2026-10-03T15:14:47Z","completed_at":"2026-10-03T17:35:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T17:44:11.0717184Z","timing_metadata_checked_at":"2026-10-03T17:44:11.0717184Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T17:44:11.075036389Z","region_metadata_checked_at":"2026-10-03T17:44:11.075036389Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"958dd735828a6f380a34b8f40e8bb7405a576cff95757bbd214a03126c2e54ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402589850472448","occurred_at":"2026-10-03T15:14:47Z","failed_at":"e2e","signature_id":"afc72a53fa7774ed1734007a58442b2c5acbb5d61e6acaf495185cd900df7590","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x1737a1124260\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kjgdvvpr4g6r\" to finish creating: GET https://rp.j0472448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"name\": \"9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T16:34:46.276618892Z\",\n \"endTime\": \"2026-10-03T16:53:53.086174722Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x1737a1124260\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kjgdvvpr4g6r\" to finish creating: GET https://rp.j0472448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"name\": \"9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T16:34:46.276618892Z\",\n \"endTime\": \"2026-10-03T16:53:53.086174722Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402405829578752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7340,"pr_state":"open","pr_sha":"e0e2b9b5f45257ab5de133a4651e3a7e85f559f0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T15:14:03Z","started_at":"2026-10-03T15:14:03Z","completed_at":"2026-10-03T17:16:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T17:23:30.858942952Z","timing_metadata_checked_at":"2026-10-03T17:23:30.858942952Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T17:23:30.865068732Z","region_metadata_checked_at":"2026-10-03T17:23:30.865068732Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0f32fc7afb0e4a5a14af2a3a7a80320a0f4e41c323870044016b5cb2b5b55af4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402405829578752","occurred_at":"2026-10-03T15:14:03Z","failed_at":"e2e","signature_id":"dca811aa20ada3f91e9119ee0eecf021dba8048b0d56a9111868dce8ce4a50a4","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x22d6ad451998\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x22d6ad451998\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"b1c110075ee77dc27f92988b75c56175ac5e389c180d06cd5a9499498d5f3510","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402405829578752","occurred_at":"2026-10-03T15:14:03Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T16:03:43Z\n Ended: 2026-10-03T16:09:42Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9578752-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106398375271206912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7326,"pr_state":"closed","pr_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","final_merged_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T14:58:02Z","started_at":"2026-10-03T14:58:02Z","completed_at":"2026-10-03T15:16:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T15:29:07.558546071Z","timing_metadata_checked_at":"2026-10-03T15:29:07.558546071Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-03T15:29:07.564085353Z","region_metadata_checked_at":"2026-10-03T16:32:06.827297578Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2c7b58542b1daa8e6ae86d29617f603a208ae41c7d97524bb6cc817bf8b8f3d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106398375271206912","occurred_at":"2026-10-03T14:58:02Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106396412521484288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7318,"pr_state":"closed","pr_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","final_merged_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T14:50:14Z","started_at":"2026-10-03T14:50:14Z","completed_at":"2026-10-03T16:56:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T17:02:50.844127768Z","timing_metadata_checked_at":"2026-10-03T17:02:50.844127768Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T17:02:50.847333458Z","region_metadata_checked_at":"2026-10-03T17:02:50.847333458Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7341/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106378931551604736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7341,"pr_state":"closed","pr_sha":"32714ac0f66cb0a9e03b0f3781d61c5812a015d5","final_merged_sha":"b870cb897e3705bbdc0b2ea937f9ccc37eee523e","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T13:40:45Z","started_at":"2026-10-03T13:40:45Z","completed_at":"2026-10-03T16:04:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T16:11:09.224238736Z","timing_metadata_checked_at":"2026-10-03T16:11:09.224238736Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T16:11:09.227116627Z","region_metadata_checked_at":"2026-10-03T16:11:09.227116627Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106365799160614912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7340,"pr_state":"open","pr_sha":"e0e2b9b5f45257ab5de133a4651e3a7e85f559f0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T12:48:35Z","started_at":"2026-10-03T12:48:35Z","completed_at":"2026-10-03T15:03:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T15:08:31.638086356Z","timing_metadata_checked_at":"2026-10-03T15:08:31.638086356Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T15:08:31.641536045Z","region_metadata_checked_at":"2026-10-03T15:08:31.641536045Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"632f2ac9558e97b5be3e0c9229021fa6a91714bd358870221b02b9886daaf2cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106365799160614912","occurred_at":"2026-10-03T12:48:35Z","failed_at":"e2e","signature_id":"b0563ce9088e2d908cb5cdc30392d78444947ed998364dd964a7182a39daa5f6","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-vzb878crbjw5\" to finish creating: GET https://rp.j0614912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"name\": \"737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T14:04:39.339033685Z\",\n \"endTime\": \"2026-10-03T14:23:40.256240312Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-vzb878crbjw5\" to finish creating: GET https://rp.j0614912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"name\": \"737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T14:04:39.339033685Z\",\n \"endTime\": \"2026-10-03T14:23:40.256240312Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106361385184137216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7318,"pr_state":"closed","pr_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","final_merged_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T12:31:03Z","started_at":"2026-10-03T12:31:03Z","completed_at":"2026-10-03T14:38:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T14:47:01.104451426Z","timing_metadata_checked_at":"2026-10-03T14:47:01.104451426Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T14:47:01.107531117Z","region_metadata_checked_at":"2026-10-03T14:47:01.107531117Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9fb13f1013352559ec9a50ccbeedb21b9bca25d368d92c390edacfc02a4f56a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106361385184137216","occurred_at":"2026-10-03T12:31:03Z","failed_at":"e2e","signature_id":"c05cf9daf6680f1855b02943b461dc08c3799199d299289700c2fc278277b2c4","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x1d83c5b77a40\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tzccwkmbcld2\" to finish creating: GET https://rp.j4137216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/63335228-135d-4644-b7d4-651deba91b70\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/63335228-135d-4644-b7d4-651deba91b70\",\n \"name\": \"63335228-135d-4644-b7d4-651deba91b70\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T13:14:48.144915783Z\",\n \"endTime\": \"2026-10-03T13:33:57.141188553Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-47kjt-tgj5p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-tzccwkmbcld2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-47kjt-tgj5p to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-47kjt-tgj5p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-tzccwkmbcld2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-47kjt-tgj5p to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x1d83c5b77a40\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-tzccwkmbcld2\" to finish creating: GET https://rp.j4137216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/63335228-135d-4644-b7d4-651deba91b70\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/63335228-135d-4644-b7d4-651deba91b70\",\n \"name\": \"63335228-135d-4644-b7d4-651deba91b70\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T13:14:48.144915783Z\",\n \"endTime\": \"2026-10-03T13:33:57.141188553Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-47kjt-tgj5p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-tzccwkmbcld2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-47kjt-tgj5p to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-47kjt-tgj5p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-tzccwkmbcld2--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-47kjt-tgj5p to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106355570880548864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7318,"pr_state":"closed","pr_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","final_merged_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T12:07:56Z","started_at":"2026-10-03T12:07:56Z","completed_at":"2026-10-03T12:23:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T12:28:43.776440724Z","timing_metadata_checked_at":"2026-10-03T12:28:43.776440724Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T12:28:43.779884112Z","region_metadata_checked_at":"2026-10-03T12:28:43.779884112Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"16469d0e96e4bc7f2e45e97673b95797758ae3e2f5450f6ce28accca19f79f2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106355570880548864","occurred_at":"2026-10-03T12:07:56Z","failed_at":"provision","signature_id":"beba06114dcaa7d8f5cf05070efb098c564f39c87ef41d58003f0a49b2fc4e73","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-03T12:18:04.762Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n Kind: Shell\\n Command: ./swift-vnet.sh\\n\"\ntime=2026-10-03T12:18:05.175Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Failed to load or parse file /tmp/azure-cli-config-3770628587/azureProfile.json. It will be overridden by default settings.\\nERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc","failure_pattern":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","search_query":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'."}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106344080505049088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7338,"pr_state":"closed","pr_sha":"e49406545480a4617927868edca9ac3c5a34842c","final_merged_sha":"e49406545480a4617927868edca9ac3c5a34842c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T11:22:17Z","started_at":"2026-10-03T11:22:17Z","completed_at":"2026-10-03T13:56:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T14:04:07.733045833Z","timing_metadata_checked_at":"2026-10-03T14:04:07.733045833Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T14:04:07.735609325Z","region_metadata_checked_at":"2026-10-03T14:04:07.735609325Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1dbe49f942a9b96ff4598ef6e59436df5c75556d57b50ae869741738f18d6132","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106344080505049088","occurred_at":"2026-10-03T11:22:17Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T12:40:19Z\n Ended: 2026-10-03T13:01:20Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5049088-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"384fe8b70bcf3555a05b604c55fa1a5693797acd7f5a1f8bdc1cee348a4483d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106344080505049088","occurred_at":"2026-10-03T11:22:17Z","failed_at":"e2e","signature_id":"3c3f2ccc36c35f1c9e21f03f7eff4418f1739e7896a5865697c85dc9064d117f","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d36ccbc2160\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vgn28b5mgscc\" to finish creating: GET https://rp.j5049088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1833854b-b6c4-421d-92ac-acbc850a2d4d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1833854b-b6c4-421d-92ac-acbc850a2d4d\",\n \"name\": \"1833854b-b6c4-421d-92ac-acbc850a2d4d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T13:01:15.160971572Z\",\n \"endTime\": \"2026-10-03T13:20:17.766960003Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq to exist\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq to exist\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d36ccbc2160\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vgn28b5mgscc\" to finish creating: GET https://rp.j5049088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1833854b-b6c4-421d-92ac-acbc850a2d4d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1833854b-b6c4-421d-92ac-acbc850a2d4d\",\n \"name\": \"1833854b-b6c4-421d-92ac-acbc850a2d4d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T13:01:15.160971572Z\",\n \"endTime\": \"2026-10-03T13:20:17.766960003Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq to exist\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-6hpdq to exist\\nMachine r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-vgn28b5mgscc--managed/providers/Microsoft.Compute/virtualMachines/r7b5q6v9r2x8a0b-autoscale-noaz-779kw-r4zl5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c3a9ccd747392dd1e1bf958e905cac99596e3fe98e783f962a633585c7962a8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106344080505049088","occurred_at":"2026-10-03T11:22:17Z","failed_at":"e2e","signature_id":"21861e38aab43579ee30401f2af52d615d1c92a4943c93ec549c760780f28b16","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x227e9786b20\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-9q2trs\" in resourcegroup=\"rg-np-skip-minor-9q2trs-89955s58px2l\" to finish creating: GET https://rp.j5049088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c34cd-a348-440a-9cce-2255b2dba56d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c34cd-a348-440a-9cce-2255b2dba56d\",\n \"name\": \"8f9c34cd-a348-440a-9cce-2255b2dba56d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T12:59:28.477851902Z\",\n \"endTime\": \"2026-10-03T13:18:37.714240199Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9q2trs-89955s58px2l-np-sm-9q2trs--managed/providers/Microsoft.Compute/virtualMachines/e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9q2trs-89955s58px2l-np-sm-9q2trs--managed/providers/Microsoft.Compute/virtualMachines/e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x227e9786b20\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-9q2trs\" in resourcegroup=\"rg-np-skip-minor-9q2trs-89955s58px2l\" to finish creating: GET https://rp.j5049088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c34cd-a348-440a-9cce-2255b2dba56d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f9c34cd-a348-440a-9cce-2255b2dba56d\",\n \"name\": \"8f9c34cd-a348-440a-9cce-2255b2dba56d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T12:59:28.477851902Z\",\n \"endTime\": \"2026-10-03T13:18:37.714240199Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9q2trs-89955s58px2l-np-sm-9q2trs--managed/providers/Microsoft.Compute/virtualMachines/e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9q2trs-89955s58px2l-np-sm-9q2trs--managed/providers/Microsoft.Compute/virtualMachines/e1y7t9q3u1x3l7k-nps-420-422-6jsw9-hhzvf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7334/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106339789761417216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7334,"pr_state":"closed","pr_sha":"df793e0d79069c2fd4aa1e8d0a96fc2e48b92451","final_merged_sha":"df793e0d79069c2fd4aa1e8d0a96fc2e48b92451","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T11:05:14Z","started_at":"2026-10-03T11:05:14Z","completed_at":"2026-10-03T13:54:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T14:04:07.554863801Z","timing_metadata_checked_at":"2026-10-03T14:04:07.554863801Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T14:04:07.55831059Z","region_metadata_checked_at":"2026-10-03T14:04:07.55831059Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106319269334618112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7318,"pr_state":"closed","pr_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","final_merged_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T09:43:41Z","started_at":"2026-10-03T09:43:41Z","completed_at":"2026-10-03T11:59:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T12:07:43.864622856Z","timing_metadata_checked_at":"2026-10-03T12:07:43.864622856Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T12:07:43.868061041Z","region_metadata_checked_at":"2026-10-03T12:07:43.868061041Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"cf72fcc6a8a8329c7bf827e3b98a700cdf365d68eafb72a87cfd8524f6be6876","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106319269334618112","occurred_at":"2026-10-03T09:43:41Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T10:35:51Z\n Ended: 2026-10-03T10:41:49Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4618112-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7268/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106316215747088384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7268,"pr_state":"open","pr_sha":"e83df41de9079bdb5b597c55cd9681d618094984","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T09:31:33Z","started_at":"2026-10-03T09:31:33Z","completed_at":"2026-10-03T11:56:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T12:07:43.46503529Z","timing_metadata_checked_at":"2026-10-03T12:07:43.46503529Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T12:07:43.468611574Z","region_metadata_checked_at":"2026-10-03T12:07:43.468611574Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fee8aa102f2d89b6f24927d984b018cb78bd5f62ef88f2013fb5ed59bcea442d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7268/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106316215747088384","occurred_at":"2026-10-03T09:31:33Z","failed_at":"e2e","signature_id":"8e852f466cf78bfd32f7a3098547e4a2279ba62592d4769dc239478e539ec985","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ccede026220\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-74v4krpx78dr\" to finish creating: GET https://rp.j7088384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61d03357-4fb7-49b2-a777-a99ffa662551\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61d03357-4fb7-49b2-a777-a99ffa662551\",\n \"name\": \"61d03357-4fb7-49b2-a777-a99ffa662551\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T10:52:21.517435656Z\",\n \"endTime\": \"2026-10-03T11:11:28.16247644Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st to exist\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st to exist\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ccede026220\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-74v4krpx78dr\" to finish creating: GET https://rp.j7088384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61d03357-4fb7-49b2-a777-a99ffa662551\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61d03357-4fb7-49b2-a777-a99ffa662551\",\n \"name\": \"61d03357-4fb7-49b2-a777-a99ffa662551\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T10:52:21.517435656Z\",\n \"endTime\": \"2026-10-03T11:11:28.16247644Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st to exist\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt to exist\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-ct7st to exist\\nMachine c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-74v4krpx78dr--managed/providers/Microsoft.Compute/virtualMachines/c0j2z3h4d3u2n8r-autoscale-noaz-5gxpz-nbhvt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106307424179720192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"closed","pr_sha":"1495f07f761e3c094aa60549d95c4cf2cf1748e9","final_merged_sha":"0d147555918839b4a556ae5981b12356ac4cf337","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-03T08:56:37Z","started_at":"2026-10-03T08:56:37Z","completed_at":"2026-10-03T11:38:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T11:46:31.91349312Z","timing_metadata_checked_at":"2026-10-03T11:46:31.91349312Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T11:46:31.91656301Z","region_metadata_checked_at":"2026-10-03T11:46:31.91656301Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7334/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106297012751175680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7334,"pr_state":"closed","pr_sha":"df793e0d79069c2fd4aa1e8d0a96fc2e48b92451","final_merged_sha":"df793e0d79069c2fd4aa1e8d0a96fc2e48b92451","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T08:15:15Z","started_at":"2026-10-03T08:15:15Z","completed_at":"2026-10-03T10:29:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T10:32:36.968058061Z","timing_metadata_checked_at":"2026-10-03T10:32:36.968058061Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T10:32:36.971898148Z","region_metadata_checked_at":"2026-10-03T10:32:36.971898148Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"e42b1436444203f19d22144aa8b53e759589613daaa489597343618125640705","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7334/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106297012751175680","occurred_at":"2026-10-03T08:15:15Z","failed_at":"alert","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T09:10:27Z\n Ended: 2026-10-03T09:28:26Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1175680-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T09:15:27Z\n Ended: 2026-10-03T09:28:26Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1175680-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106283184227356672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7318,"pr_state":"closed","pr_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","final_merged_sha":"0b3f888481a17781af6aac00a9db6f1c4fc58df9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T07:20:18Z","started_at":"2026-10-03T07:20:18Z","completed_at":"2026-10-03T09:38:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T09:50:21.000624046Z","timing_metadata_checked_at":"2026-10-03T09:50:21.000624046Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T09:50:21.003638035Z","region_metadata_checked_at":"2026-10-03T09:50:21.003638035Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"8a1bc5b64cfc6a39fac7805c805c1e051d5f1c84b09024f43f0817ee0aeb74b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106283184227356672","occurred_at":"2026-10-03T07:20:18Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T08:23:18Z\n Ended: 2026-10-03T08:30:15Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7356672-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232246682783744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7336,"pr_state":"closed","pr_sha":"0c82ea710333dd869bbd1a24686f11e37cfe9b99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T03:57:54Z","started_at":"2026-10-03T03:57:54Z","completed_at":"2026-10-03T06:21:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T06:42:21.214113117Z","timing_metadata_checked_at":"2026-10-03T06:42:21.214113117Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T06:42:21.217856805Z","region_metadata_checked_at":"2026-10-03T06:42:21.217856805Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"10fed181611039bb684ee91aa7d3fa3ec49d7fd995949d165d2d931b8a9493d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232246682783744","occurred_at":"2026-10-03T03:57:54Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T05:06:36Z\n Ended: 2026-10-03T05:12:36Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2783744-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"dbf50194192ece1b1a6409cf6bdaea3b43fe3b2554d4508bd2440e79909d3914","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232246682783744","occurred_at":"2026-10-03T03:57:54Z","failed_at":"e2e","signature_id":"4ecba326214d91b223bb8a28fa503efb6110b5f309f19014e5eec742eee54784","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x1229c1294580\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-q4mxjczq7tsd\" to finish creating: GET https://rp.j2783744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7757c823-88ba-458e-aed3-9dfdac2d7080\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7757c823-88ba-458e-aed3-9dfdac2d7080\",\n \"name\": \"7757c823-88ba-458e-aed3-9dfdac2d7080\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T05:03:30.952777318Z\",\n \"endTime\": \"2026-10-03T05:22:34.856876402Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-62mv4-kp8nq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-q4mxjczq7tsd--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-62mv4-kp8nq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-62mv4-kp8nq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-q4mxjczq7tsd--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-62mv4-kp8nq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0x1229c1294580\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-q4mxjczq7tsd\" to finish creating: GET https://rp.j2783744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7757c823-88ba-458e-aed3-9dfdac2d7080\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7757c823-88ba-458e-aed3-9dfdac2d7080\",\n \"name\": \"7757c823-88ba-458e-aed3-9dfdac2d7080\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T05:03:30.952777318Z\",\n \"endTime\": \"2026-10-03T05:22:34.856876402Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-62mv4-kp8nq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-q4mxjczq7tsd--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-62mv4-kp8nq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-62mv4-kp8nq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-q4mxjczq7tsd--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-62mv4-kp8nq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232206048366592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7335,"pr_state":"closed","pr_sha":"b25f2cbc29ff36f009c24cff5cd46c29b5a264ab","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T03:57:44Z","started_at":"2026-10-03T03:57:44Z","completed_at":"2026-10-03T06:19:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T06:31:30.403522951Z","timing_metadata_checked_at":"2026-10-03T06:31:30.403522951Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T06:31:30.40698534Z","region_metadata_checked_at":"2026-10-03T06:31:30.40698534Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b08aefbb1959664a9df2d569690d04010f1efa0b569d2db1644f86f9768ee6d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232206048366592","occurred_at":"2026-10-03T03:57:44Z","failed_at":"e2e","signature_id":"4cb0132308929f56cf10c7cf8dcc82eb33338d500ae0ff69d7f2b06b20772f9b","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x331a5ee39be0\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-hzfdbcnlckj2\" to finish creating: GET https://rp.j8366592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7eb0be6-172b-405f-8019-aa4a01912e9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7eb0be6-172b-405f-8019-aa4a01912e9a\",\n \"name\": \"f7eb0be6-172b-405f-8019-aa4a01912e9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T05:21:56.267331547Z\",\n \"endTime\": \"2026-10-03T05:41:00.127542846Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-hzfdbcnlckj2--managed/providers/Microsoft.Compute/virtualMachines/f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-hzfdbcnlckj2--managed/providers/Microsoft.Compute/virtualMachines/f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x331a5ee39be0\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-hzfdbcnlckj2\" to finish creating: GET https://rp.j8366592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7eb0be6-172b-405f-8019-aa4a01912e9a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7eb0be6-172b-405f-8019-aa4a01912e9a\",\n \"name\": \"f7eb0be6-172b-405f-8019-aa4a01912e9a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T05:21:56.267331547Z\",\n \"endTime\": \"2026-10-03T05:41:00.127542846Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-hzfdbcnlckj2--managed/providers/Microsoft.Compute/virtualMachines/f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-hzfdbcnlckj2--managed/providers/Microsoft.Compute/virtualMachines/f3y4c6b9n5p5z6y-cordon-np-qb8hf-8gc5p to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106223756383883264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7332,"pr_state":"open","pr_sha":"3fd6df7d8888145d8258955876aeb97dfd0efbb8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T03:24:09Z","started_at":"2026-10-03T03:24:09Z","completed_at":"2026-10-03T05:51:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T06:10:10.214122465Z","timing_metadata_checked_at":"2026-10-03T06:10:10.214122465Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T06:10:10.218235452Z","region_metadata_checked_at":"2026-10-03T06:10:10.218235452Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"152f667948b6ca5fd63b8ccff02f563599d770f85f69db6a6541aebf8c261bee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106223756383883264","occurred_at":"2026-10-03T03:24:09Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T04:38:45Z\n Ended: 2026-10-03T04:44:46Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3883264-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"6fccb2861b1ecbef1d1d2954b420844ad3d2eda4e3ea3e72b320c03b8674b140","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106223756383883264","occurred_at":"2026-10-03T03:24:09Z","failed_at":"alert","signature_id":"fee414b15571075bb38e6218ae18960a82bae3ac8dd2273bf21bea06f37d99a3","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-03T04:27:33Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j3883264-svc\", component=\"backend\", name=\"openshiftversioncatalog\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue openshiftversioncatalog has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"c44920fe390496e7de07f1b46f2a57881e5bb54239316b8d36e46c7b09ad07f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106223756383883264","occurred_at":"2026-10-03T03:24:09Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106216516587360256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7333,"pr_state":"closed","pr_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","final_merged_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T02:55:23Z","started_at":"2026-10-03T02:55:23Z","completed_at":"2026-10-03T05:20:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T05:38:36.895533037Z","timing_metadata_checked_at":"2026-10-03T05:38:36.895533037Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T05:38:36.899385524Z","region_metadata_checked_at":"2026-10-03T05:38:36.899385524Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5bbb640259ead5e9f3e0b0d6fc954c806d2bde688e66f62cafc276c85df15087","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106216516587360256","occurred_at":"2026-10-03T02:55:23Z","failed_at":"e2e","signature_id":"405132e552ca41557cf8c6f086fdcc0c25a7a484630f63163bb4bed7e7d238ad","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0x1113bcb9c160\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-rckbfw\" in resourcegroup=\"rg-np-downgrade-rckbfw-856zsqvm5rkb\" to finish creating: GET https://rp.j7360256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/074aacd3-7322-43b0-9305-f06220347ff4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/074aacd3-7322-43b0-9305-f06220347ff4\",\n \"name\": \"074aacd3-7322-43b0-9305-f06220347ff4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T04:27:23.785116321Z\",\n \"endTime\": \"2026-10-03T04:46:25.26888513Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-rckbfw-856zsqvm5rkb-np-dg-rckbfw--managed/providers/Microsoft.Compute/virtualMachines/l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-rckbfw-856zsqvm5rkb-np-dg-rckbfw--managed/providers/Microsoft.Compute/virtualMachines/l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0x1113bcb9c160\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-rckbfw\" in resourcegroup=\"rg-np-downgrade-rckbfw-856zsqvm5rkb\" to finish creating: GET https://rp.j7360256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/074aacd3-7322-43b0-9305-f06220347ff4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/074aacd3-7322-43b0-9305-f06220347ff4\",\n \"name\": \"074aacd3-7322-43b0-9305-f06220347ff4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T04:27:23.785116321Z\",\n \"endTime\": \"2026-10-03T04:46:25.26888513Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-rckbfw-856zsqvm5rkb-np-dg-rckbfw--managed/providers/Microsoft.Compute/virtualMachines/l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-rckbfw-856zsqvm5rkb-np-dg-rckbfw--managed/providers/Microsoft.Compute/virtualMachines/l7r8a0n5m4k8l1t-npdg-4-21-z7xhd-fq8sk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cd4eb016a6af8364f67921f741d6d11977f17449764ccf3e8e4e9a9cbbb080e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106216516587360256","occurred_at":"2026-10-03T02:55:23Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T04:07:22Z\n Ended: 2026-10-03T04:13:25Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7360256-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106201793171034112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7331,"pr_state":"closed","pr_sha":"53a4ed21d7b546b4f4ac2fb406d7d9bec086289b","final_merged_sha":"d5ee24cc09296f5a16b33a9dbe4bd808f7e62ec2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T01:56:53Z","started_at":"2026-10-03T01:56:53Z","completed_at":"2026-10-03T04:25:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T04:35:41.037657768Z","timing_metadata_checked_at":"2026-10-03T04:35:41.037657768Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T04:35:41.042354053Z","region_metadata_checked_at":"2026-10-03T04:35:41.042354053Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"977a2c03fc29820644e6afcb40430d4dd99b5781670499f0c4ab041c051a2379","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106201793171034112","occurred_at":"2026-10-03T01:56:53Z","failed_at":"alert","signature_id":"fa412034903709961f37b7b9179b2905b4c4fb3d1b051e1833d51c25494e7359","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T03:07:49Z\n Ended: 2026-10-03T03:19:50Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1034112-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T03:08:45Z\n Ended: 2026-10-03T03:22:45Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1034112-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106186045094629376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7335,"pr_state":"closed","pr_sha":"b25f2cbc29ff36f009c24cff5cd46c29b5a264ab","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T00:54:18Z","started_at":"2026-10-03T00:54:18Z","completed_at":"2026-10-03T03:16:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T03:32:21.511611898Z","timing_metadata_checked_at":"2026-10-03T03:32:21.511611898Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T03:32:21.515623085Z","region_metadata_checked_at":"2026-10-03T03:32:21.515623085Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"07695946dec4d21ca6bb36750251bb90756265e3fa3ee37e4dae4e6486c1e5de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106186045094629376","occurred_at":"2026-10-03T00:54:18Z","failed_at":"alert","signature_id":"fa412034903709961f37b7b9179b2905b4c4fb3d1b051e1833d51c25494e7359","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T01:56:32Z\n Ended: 2026-10-03T02:03:33Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4629376-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T01:57:33Z\n Ended: 2026-10-03T02:10:32Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4629376-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"3b3e7b53ab568a28c1b36ad2580873415bbd2c69f08b74aca967f8a4556e68b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106186045094629376","occurred_at":"2026-10-03T00:54:18Z","failed_at":"e2e","signature_id":"5bfc63de429111a26e0e3c0120b6821322754898f203dabd0929d69d6732c4f1","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3263781c03a0\u003e: \n DNS for route host agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3263781c03a0\u003e: \n DNS for route host agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106184864796512256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7336,"pr_state":"closed","pr_sha":"0c82ea710333dd869bbd1a24686f11e37cfe9b99","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-03T00:49:37Z","started_at":"2026-10-03T00:49:37Z","completed_at":"2026-10-03T03:19:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T03:32:21.69026701Z","timing_metadata_checked_at":"2026-10-03T03:32:21.69026701Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T03:32:21.692941801Z","region_metadata_checked_at":"2026-10-03T03:32:21.692941801Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0dba5f7c0186f27a7716c513c7f465b1f6ca6155f811f6c3c341b55d7094071d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106184864796512256","occurred_at":"2026-10-03T00:49:37Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"0f550217af47207c5f9de35b57e8c8dc3b2e55e73691648da225c96058fa2c4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106184864796512256","occurred_at":"2026-10-03T00:49:37Z","failed_at":"e2e","signature_id":"12b342095018d1a5c43a8d0f0915ddf5043eaa8b66110e00ae45d0793de2806b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-xt2c7s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b2c75ad8a80\u003e: \n DNS for route host agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-xt2c7s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b2c75ad8a80\u003e: \n DNS for route host agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"df977953ff39d5d045e2981e35f026ff36bc89407bfbb1f3db17e4f038cdc89e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106184864796512256","occurred_at":"2026-10-03T00:49:37Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T02:01:45Z\n Ended: 2026-10-03T02:07:46Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6512256-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106183789494407168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7333,"pr_state":"closed","pr_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","final_merged_sha":"b4fa7c165d2869972e6d168a8f42b45697830cb5","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T00:45:21Z","started_at":"2026-10-03T00:45:21Z","completed_at":"2026-10-03T02:03:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T02:18:30.352206704Z","timing_metadata_checked_at":"2026-10-03T02:18:30.352206704Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T02:18:30.355040096Z","region_metadata_checked_at":"2026-10-03T02:18:30.355040096Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"5b98b67a95c3198431956740459473f5d79d9c63aabfee380becbad2411b3aa8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106183789494407168","occurred_at":"2026-10-03T00:45:21Z","failed_at":"provision","signature_id":"f292884abfee87bf70d297d035f626393e7b29fe680eeec4ec5586d3b8b28c52","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"templatize-pipeline","failure_text":"time=2026-10-03T01:35:28.208Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-03T01:35:28.778Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy values=\"map[azureArmHelperIdentityCertName:armHelperPoolCert-10 azureArmHelperIdentityClientId:0a9e3940-6bc3-4cbf-a8f7-c65c3d8d2462 azureArmHelperMockFpaPrincipalId:47f69502-0065-4d9a-b19b-d403e183d2f4 azureCloudEnvironmentName:AzurePublicCloud azureCsMiClientId:de4f7be2-47e3-4002-adc2-77c2fcdec830 azureFirstPartyApplicationClientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX azureMiMockServicePrincipalCertName:msiMockPoolCert-17 azureMiMockServicePrincipalClientId:e7112f7f-5b45-480e-bc44-abf9f219e6f0 azureMiMockServicePrincipalPrincipalId:39eefc4f-5e24-491d-9f09-bcc0ad573bcf azureOperatorsMI:map[dev:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] public:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] roleSetName:dev] backplaneURL: batchProcesses: batchProcessesDryRun:true clientId:foo clientScopes:openid clientSecret:bar clusterErrorReport:false cosmosDatabaseName:arohcpci01-rp-j4407168 cosmosDatabaseUrl:https://arohcpci01-rp-j4407168.documents.azure.com:443/ cpuRequest:630m csDeploymentStrategy:map[rollingUpdate:map[maxSurge:1 maxUnavailable:0] type:RollingUpdate] dataPlaneHAProxyImage:arohcpocpdev.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678 databaseAuthMethod:postgres databaseDisableTls:true databaseHost:ocm-cs-db databaseK8sSecretName:ocm-cs-db databaseName:ocm-cs-db databasePassword:TheBlurstOfTimes databasePort:5432 databaseServiceName:ocm-cs-db databaseUser:ocm databaseVolumeCapacity:512Mi debugMaxDuration:5m debugPort: defaultExpiration:0 deployment:map[zoneCount:3] environment:arohcpci01 forceMigration: fpaCertName:firstPartyCert2 gatewayUrl:http://127.0.0.1:9090 global:map[imageDigest:sha256:5fffce295f8b49be38ae2385714d09af29700dd3dc60c40a058f7aeb66e97d72] imageDigest:sha256:5fffce295f8b49be38ae2385714d09af29700dd3dc60c40a058f7aeb66e97d72 imageRegistry:arohcpsvcdev.azurecr.io imageRepository:app-sre/aro-hcp-clusters-service insecure:false jwksUrl:http://localhost logLevel:4 machinePoolMigrationWorkerPeriod:1h managedIdentitiesDataPlaneAudienceResource:https://dummy.org maximumExpiration:0 memoryLimit:unlimited memoryRequest:150Mi ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev ocpAcrResourceUrl:arohcpocpdev.azurecr.io oidcIssuerBaseUrl:https://arohcpci01oidcj4407168.z9.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj4407168.blob.core.windows.net/ postgresqlVersion:12 provisionShardClusterLimit:100 pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:app-sre/aro-hcp-clusters-service:pull workloadIdentityClientId:27f533be-4c23-4032-9d19-da9c7454bd3b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] region:canadacentral replicas:3 runtimeMode:aro-hcp serviceAccountName:clusters-service serviceKeyVaultName:aro-hcp-dev-svc-kv tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesEnabled:true tlsCertificatesIssuer:Self tokenUrl:http://localhost useAzureDB:false userDefinedDnsBaseDomain:i1.devshift.org]\"\ntime=2026-10-03T01:35:28.779Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-03T01:35:29.089Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy namespace=clusters-service\ntime=2026-10-03T01:35:29.415Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy dryRun=false\ntime=2026-10-03T01:35:29.416Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-03T01:35:29.465Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-03T01:45:35.236Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-10-03T01:45:35.237Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a8bb4542e08df91dbf01364e592eda2b949d63ea36dbf43a83cbdbd4c51b7dfe","failure_pattern":"failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded"}},{"row_id":"6666431c752f574bfbdf44722e36ce746442fea8bf646ae33fad2cd76e67b21f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106183789494407168","occurred_at":"2026-10-03T00:45:21Z","failed_at":"alert","signature_id":"5772227d0b1c3dbc1f88ff19aefa92151e392000e5bb31a2f32a22c36981ef0c","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-03T01:40:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4407168-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-74695d59f9-hxdpn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-74695d59f9-hxdpn has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-03T01:42:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4407168-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-74695d59f9-bfnvs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-74695d59f9-bfnvs has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-03T01:42:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4407168-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-74695d59f9-wjv8w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-74695d59f9-wjv8w has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","a8bb4542e08df91dbf01364e592eda2b949d63ea36dbf43a83cbdbd4c51b7dfe"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"prod","summary":{"total_runs":6,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T23:00:45Z","started_at":"2026-10-03T23:00:45Z","completed_at":"2026-10-04T00:39:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-04T00:47:38.949027707Z","timing_metadata_checked_at":"2026-10-04T00:47:38.949027707Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-04T00:47:38.952123897Z","region_metadata_checked_at":"2026-10-04T00:47:38.952123897Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"19a372277c7dece463b21300f514a070e5bef1b43d45c7b41d79e08ac45a436e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"467ff9b3fff0fba450d4251bf1cedce098cd2ea5e773aae9c7dab251c9546c13","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-h26qrg\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n \u003c*exported.ResponseError | 0x34bdd7d932c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-5fxfbtp77wdj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-h26qrg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-h26qrg\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n \u003c*exported.ResponseError | 0x34bdd7d932c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-5fxfbtp77wdj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-h26qrg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2c9acf0a041e35db9a096a5e9a1c6e5fde6548c33d99abc709b51e8f0e31e3b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"eb95f17360545876d289dee37a778611d4344608288ffa82220daa2648023378","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x156bbec7cb0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-48mh6dg9zlr8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x156bbec7cb0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-48mh6dg9zlr8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2e5733bb82a9d59382f96eebdd4b8d060ddc95729d17fc114d2de16fdd18782d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"b1e65d263ab4ed8d6d367a1d2b0e22cf65bd386883e59e308c33eef10b3fdd3c","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x210c33ec90b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8fsbbrnx57fx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x210c33ec90b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8fsbbrnx57fx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"47c1e63ee81b5efdfd595d023a3110e0ef02a7c95f3c70194bafd2f049aebe23","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"46e00fd4ba016237d54a5c172dba5d7ae1620c42bcb5d60c65e763e9df2f79cd","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xd97df9bad80\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cskpwn77dkwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xd97df9bad80\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cskpwn77dkwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"511207853c6006a6c2ead9c333f6e259cb9ed0a167e05a61432a0e5bf3ba16d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"6cf60cc34982ca354ed254e7db99f8c3c17c9ef039ff68ffc95b5e18a6f44b25","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x781abe80140\u003e: \n failed to create NodePool np-4kgdcw: failed starting nodepool creation \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\" in resourcegroup=\"rg-nightly-4-20-4t8l44cr8nbr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-4t8l44cr8nbr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-4kgdcw/nodePools/np-4kgdcw\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x781abe80140\u003e: \n failed to create NodePool np-4kgdcw: failed starting nodepool creation \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\" in resourcegroup=\"rg-nightly-4-20-4t8l44cr8nbr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-4t8l44cr8nbr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-4kgdcw/nodePools/np-4kgdcw\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8783ccf2993e5a87020d3f4664b66da2f07940cec670f0bb57155f301cf26316","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"df520e23539da9e8ef8e4034c818cfb36af7d49a348b14cd808580a60fa3775f","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x2d7c17881b00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-hn22fz5qv7wv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x2d7c17881b00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-hn22fz5qv7wv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ac2b9a0207d941d433eab33b72bf782827e619c5ef3461ccd1badd6b9be82f4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"8128113059a95965bef3d3c445b7c0cac5fe7442cb1f7985e9426b72d3507e7d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n \u003c*errors.joinError | 0x29a4ab6ac030\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n \u003c*errors.joinError | 0x29a4ab6ac030\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized"}},{"row_id":"bf44aacc67413eb15a9fa079b1b4221883bf2f5cc4919481437a4e28a76d6108","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"8af9e2c89645a4bf3765a33eab4e694ab3d6205ee2dba4b2403cbbe04e4bb04b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x31d5efc20c00\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-b4wh48\" in resourcegroup=\"rg-np-version-upgrade-b4wh48-bzk7j574q45w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b4wh48-bzk7j574q45w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-b4wh48/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x31d5efc20c00\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-b4wh48\" in resourcegroup=\"rg-np-version-upgrade-b4wh48-bzk7j574q45w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b4wh48-bzk7j574q45w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-b4wh48/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c3f8e18fe302ce62947bf15e89418b8cd8f5154bb0f0602106f2189584012eae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"d23e0af5f66a9d21ebfe7f67bab5ec265f1980b1c4649c6850acb948a72ccc0c","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-54csls\" to 4.21.0-0.nightly-multi-2026-10-02-115050\nUnexpected error:\n \u003c*exported.ResponseError | 0x2086109860c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-hx8x6fq8plbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-54csls\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-54csls\" to 4.21.0-0.nightly-multi-2026-10-02-115050\nUnexpected error:\n \u003c*exported.ResponseError | 0x2086109860c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-hx8x6fq8plbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-54csls\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c6b165759bfdeca8f65ae4807ff00138c6e9f7674c570efc571d070ad4707f7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"ab51303944522693c0b33104b7c92d86be3eb8c3466ad20546a9b77e031ae5b1","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cb7bb3ff960\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pf8r4pxpb5hl\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-pf8r4pxpb5hl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cb7bb3ff960\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pf8r4pxpb5hl\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-pf8r4pxpb5hl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"dc35506b62aed78f873b23aa0ab86637435f48eb9d5ff5bde724c455df7fde2a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"103fa735d194341e5747569e803885dd3d43db0551359672451398c565ccf594","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xcf8828ab170\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-crv2w79vmlxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xcf8828ab170\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-crv2w79vmlxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"dfb2851a4812946146384b0bcd660048a5e5e45291bd7367319187e07771647e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"4cf3eda2914d3b1a9083e46adec66b3f55112482155fd5d64fc7f418ec60e0b0","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x325c79dc0060\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-987nc9dd4vhq\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-987nc9dd4vhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x325c79dc0060\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-987nc9dd4vhq\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-987nc9dd4vhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106310818076299264","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T09:10:07Z","started_at":"2026-10-03T09:10:07Z","completed_at":"2026-10-03T12:00:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T12:07:44.065642534Z","timing_metadata_checked_at":"2026-10-03T12:07:44.065642534Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T12:07:44.06861542Z","region_metadata_checked_at":"2026-10-03T12:07:44.06861542Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106310690099695616","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T09:09:36Z","started_at":"2026-10-03T09:09:36Z","completed_at":"2026-10-03T11:49:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T11:57:20.514770193Z","timing_metadata_checked_at":"2026-10-03T11:57:20.514770193Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T11:57:20.51866138Z","region_metadata_checked_at":"2026-10-03T11:57:20.51866138Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"5c45c8cb2cb6d588bdcb890095f27ed5f60f4fa514ca17570582276db85ad95d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106310690099695616","occurred_at":"2026-10-03T09:09:36Z","failed_at":"e2e","signature_id":"3bc3377f053adf5e02635272297b8adacea15f1260ea4f1c4bf49a21872672c8","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2861f94db110\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-s8tjmf2mg47r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"name\": \"a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:45:52.709942431Z\",\n \"endTime\": \"2026-10-03T10:04:55.390706618Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-mf948-nwnch: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-mf948-nwnch: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2861f94db110\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-s8tjmf2mg47r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"name\": \"a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:45:52.709942431Z\",\n \"endTime\": \"2026-10-03T10:04:55.390706618Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-mf948-nwnch: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-mf948-nwnch: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"743311f7cf757f8f0a1a9074c2916984b50d656001d8540e786322767e33a3ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106310690099695616","occurred_at":"2026-10-03T09:09:36Z","failed_at":"e2e","signature_id":"13af940fe84aa9225f10173afd7fedb188865ae08fb26ab20d44731489099168","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33e8cc763a20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wrvrktrj9pn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"name\": \"a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:41:16.318835994Z\",\n \"endTime\": \"2026-10-03T10:00:22.618172935Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33e8cc763a20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wrvrktrj9pn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"name\": \"a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:41:16.318835994Z\",\n \"endTime\": \"2026-10-03T10:00:22.618172935Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106309368474505216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-03T09:04:21Z","started_at":"2026-10-03T09:04:21Z","completed_at":"2026-10-03T11:38:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T11:46:32.104250608Z","timing_metadata_checked_at":"2026-10-03T11:46:32.104250608Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T11:46:32.106568501Z","region_metadata_checked_at":"2026-10-03T11:46:32.106568501Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106302957979963392","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T08:38:53Z","started_at":"2026-10-03T08:38:53Z","completed_at":"2026-10-03T11:21:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T11:25:37.799730289Z","timing_metadata_checked_at":"2026-10-03T11:25:37.799730289Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T11:25:37.802785579Z","region_metadata_checked_at":"2026-10-03T11:25:37.802785579Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c9b7b2ace7f478670b02aa26aba828b8e4f1c801082feb818e55bb81523fa333","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106302957979963392","occurred_at":"2026-10-03T08:38:53Z","failed_at":"e2e","signature_id":"adee8926a72629f9d64b7a77f2665eb3670a4af4a352e102d836fb3757b3bc93","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3a380b9ff820\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jdf825hs8fn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"name\": \"1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T10:22:50.222769152Z\",\n \"endTime\": \"2026-10-03T10:41:50.376552801Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3a380b9ff820\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jdf825hs8fn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"name\": \"1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T10:22:50.222769152Z\",\n \"endTime\": \"2026-10-03T10:41:50.376552801Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jdf825hs8fn9\" to finish creating: GET https://management"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T02:00:44Z","started_at":"2026-10-03T02:00:44Z","completed_at":"2026-10-03T04:05:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T04:14:38.273664393Z","timing_metadata_checked_at":"2026-10-03T04:14:38.273664393Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T04:14:38.277027182Z","region_metadata_checked_at":"2026-10-03T04:14:38.277027182Z"},"failed_at":["e2e"],"failed_test_count":22,"occurrences":[{"row_id":"0c9dcf6dab558084a2a9ef2ffb5928acaae8895b52a9a68a3083498e01c99d34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"7b4cf1cf1f356f8a0bd6bfb20a926fe794e222e24e9098fbe0eb9156b1ec6573","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-v6zlk7\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x3aa50a408f30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-7fls5z997c4n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-v6zlk7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-v6zlk7\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x3aa50a408f30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-7fls5z997c4n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-v6zlk7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"10bf897673aab557e061392b4e7b085cf06cf71d2e87436b087d4d916dcdf7a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"366a1991051b1a37601646ae4edff31e67686f7599119c046ea638bd894e910e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-msl8ps\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x21a7a658f5c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-l7wmbxqck8gf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-msl8ps\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-msl8ps\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x21a7a658f5c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-l7wmbxqck8gf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-msl8ps\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1575987683f490cd4bed1fc8f8ae7fbc66348f4139e4c35f3fe89965b1730630","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"f4f3787f7980560a701583c4b17b2aa434c0cb2c95496fa5661ef575b0ba94d2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-9hrm6z\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0x1415c20a5500\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-cck68gj6qfsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-9hrm6z\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-9hrm6z\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0x1415c20a5500\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-cck68gj6qfsp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-9hrm6z\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"15bdae944280f7615b0ec41135ff01e6844852a38792e237a7195cecfaaa88db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"8c999722171344de70dda1803d5579aed1e0dbc4ecd2b2c5ea3d0e6b792cbf9c","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x61296253c20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-96mcz9ktg57h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x61296253c20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-96mcz9ktg57h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1773cbbaa075a165279663c958753492642f49599aee70aebc5d049d9b8b3d87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"c6f98fc883c08bd20a81c2a2a9718e59825e4839ac864d806b4647a01ab008bd","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-x4v9ss\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x5a0752a7c50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-9j7fg4k8csln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-x4v9ss\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-x4v9ss\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x5a0752a7c50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-9j7fg4k8csln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-x4v9ss\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"18fb1a91b8a224a94f4434293ef344187f48338df84d363c146ff6a9e4f45bce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"28086d43cfa72152275388034cc851903b47aa1ca5775b0f6fa80ca38a6ab4fb","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ed03c0ba1c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-nrr89q\" resourcegroup=\"rg-np-dg-minor-nrr89q-d5btp9lwtkwk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-nrr89q-d5btp9lwtkwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-nrr89q/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ed03c0ba1c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-nrr89q\" resourcegroup=\"rg-np-dg-minor-nrr89q-d5btp9lwtkwk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-nrr89q-d5btp9lwtkwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-nrr89q/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3f4aac6bf9d1dd700cf6ea0451b03f1030eff54fe6dd1fd9c78540e5dcd9dea2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"7346747ba389cbf83d28f49662dfd4c31ae1261a21f1a9894630a81cb52c0065","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-w78qcb\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x39bf51e7ba70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-l8ksqxqxvhll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-w78qcb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-w78qcb\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0x39bf51e7ba70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-l8ksqxqxvhll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-w78qcb\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"697309d8df16b27753c6b0671e6b829487d36943157b8d477fabeff5b5aea0bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"c929d59edb7567fa76ef70a100f78549e067b5211d3fa8ccbe6bb931bf8ae3bc","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n \u003c*fmt.wrapError | 0x6bee01cdac0\u003e: \n failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-6td8ds\" resourcegroup=\"rg-np-skip-minor-6td8ds-d2dvtglcn2l2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-6td8ds-d2dvtglcn2l2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-6td8ds/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n \u003c*fmt.wrapError | 0x6bee01cdac0\u003e: \n failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-6td8ds\" resourcegroup=\"rg-np-skip-minor-6td8ds-d2dvtglcn2l2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-6td8ds-d2dvtglcn2l2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-6td8ds/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"70c71db71d474d600450c9194163e989c401e548db8d5bd50fecdaf31d395fde","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"5bb1d3c7e546810e558daf11bf222544e4133eca40ba362ffb48b7b1423ad104","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x177ce192fa20\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-z4pwtb\" resourcegroup=\"rg-np-version-upgrade-z4pwtb-4qn5bp5csjzb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z4pwtb-4qn5bp5csjzb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-z4pwtb/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x177ce192fa20\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-z4pwtb\" resourcegroup=\"rg-np-version-upgrade-z4pwtb-4qn5bp5csjzb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z4pwtb-4qn5bp5csjzb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-z4pwtb/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"76315c6f7dd1f60bcbe1892d73ecb0525546f62b7ea9847f1227a533b1e818d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"a6f43bb89c4cb869e470fa6b8f415931a9ffca73b7a099b8d5d5fdb88da93883","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x143a1310b240\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-rc8d5d\" resourcegroup=\"rg-np-version-upgrade-rc8d5d-w2t72x2zshww\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-rc8d5d-w2t72x2zshww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-rc8d5d/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0x143a1310b240\u003e: \n failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-rc8d5d\" resourcegroup=\"rg-np-version-upgrade-rc8d5d-w2t72x2zshww\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-rc8d5d-w2t72x2zshww/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-rc8d5d/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"797713a3285ba10190c71961df753255912c25d035e20b13982126106b0a503b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"1a76286ada45eadee61eadc06e02438d66934a94c715501b79fa55ea647992ba","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-w2grzz\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x30efed875560\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-52rk6bpxrc9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-w2grzz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-w2grzz\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x30efed875560\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-52rk6bpxrc9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-w2grzz\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7983ab0a2937940d5d7eb2f22a2828fbb126c6caf70b70fb774ba5fce7e5e5b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"0d1682a2cfbb5f2e590b6b31ccc2bdd35b4fb6efe346454c0a0465b4a4c377dc","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x148c3b6e47b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-tvg824p96xln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x148c3b6e47b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-tvg824p96xln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8ccac32d6425dc4004b10368b148c88ca26fe55f64ca66f8d0fb3f5fb30b3b0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"d46d14f8b1e583f77549aa7ebed41d05be17337007389372864730bed2620d7f","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x95d711c6660\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fqfljmzwnn7j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x95d711c6660\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-fqfljmzwnn7j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8f411c621c298058ef598a08dced662516f7c8f3bcf1292c64ecac83f75d34b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"6db71a6e85dcae0f4cf2fff89193035c7c77e8c6a0d75a696207a46ff9701f52","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x16c8ce6032a0\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-4rvxbgx64wn7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4rvxbgx64wn7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x16c8ce6032a0\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-4rvxbgx64wn7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4rvxbgx64wn7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"91e2e721a9f657bba3bd5ae357910154adbe8144e749f0d75326e400b961e6e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"c17fe792fb98b2d6a545cd9ddbd5a4c5a6ed15360c664292bd1e61a9ec602478","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-2bw2nd\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0x3e7c72aaef90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-h5wrz4q9tk28/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-2bw2nd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-2bw2nd\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0x3e7c72aaef90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-h5wrz4q9tk28/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-2bw2nd\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a71b9f02e0ad43c68c7551662ed1ed112c875525e01a7091c48f1ff1ce969b3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"919aee9941c508698346026ce9e0051090805691e68054cfc4acc83d6c2fb7d3","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-4c45hm\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1c31b53c8540\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-7jn7lrmbjhwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-4c45hm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-4c45hm\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1c31b53c8540\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-7jn7lrmbjhwn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-4c45hm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b264154137a8aa708b1d0300fc426cbfe7ce683d9f7bcb626e6aa0fb56875b78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"9af200308adea58cb657b883d457e0c1c8c4b68df178d893c6ee77bf70f5c283","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x18d8f5386060\u003e: \n failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-xvr7qd\" resourcegroup=\"rg-np-noedge-xvr7qd-fdgmzp5mlmp2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xvr7qd-fdgmzp5mlmp2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-xvr7qd/nodePools/npnoedge-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x18d8f5386060\u003e: \n failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-xvr7qd\" resourcegroup=\"rg-np-noedge-xvr7qd-fdgmzp5mlmp2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xvr7qd-fdgmzp5mlmp2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-xvr7qd/nodePools/npnoedge-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b7cf4cd93fde135d4224e2903f03f401dd58d99b926a5fc63f1ab451e7ae56fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"c3e02910fd51fc69091b5e63232a999211acc1001c4336ec9c326557d10e1d3a","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xa952f808ee0\u003e: \n failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-x47s9n\" resourcegroup=\"rg-np-downgrade-x47s9n-djktngm9cgxt\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-x47s9n-djktngm9cgxt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-x47s9n/nodePools/npdg-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xa952f808ee0\u003e: \n failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-x47s9n\" resourcegroup=\"rg-np-downgrade-x47s9n-djktngm9cgxt\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-x47s9n-djktngm9cgxt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-x47s9n/nodePools/npdg-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bce71be1223ae0a8de29cc53e81adc386c029a8381a038187dc4e9073274c3b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"3906d99cd657415581fb6ba0344b0053b97db85bb68b94222e88264810fdd66b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-95bqxj\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1aa712d69710\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b5g72gdsstzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-95bqxj\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-95bqxj\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1aa712d69710\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b5g72gdsstzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-95bqxj\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c18b1ae301dc8e50ce54f88d113b3e1f662ccbbe2b68de097fc69a2ae8096cf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"567f7755bb3e007fee5b222ee1274b9e42bd25b0c9f732e516cf41596ecc8fab","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0cb099f2c0\u003e: \n failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-cvfvg5\" resourcegroup=\"rg-np-version-upgrade-cvfvg5-r74zmmc476nw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-cvfvg5-r74zmmc476nw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-cvfvg5/nodePools/npupgrade-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0cb099f2c0\u003e: \n failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-cvfvg5\" resourcegroup=\"rg-np-version-upgrade-cvfvg5-r74zmmc476nw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-cvfvg5-r74zmmc476nw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-cvfvg5/nodePools/npupgrade-4-21\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d314dd0c9769760148bae161c2ecb5dc416965037b20b77b772f1612a9233d42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"44dfa2bda4098c665bbfb261786dda1e03ddea5270c9178a60475577c209a9b5","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x110fe0e8c640\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djddxfxjp2wr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-djddxfxjp2wr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x110fe0e8c640\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djddxfxjp2wr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-djddxfxjp2wr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f220b6975bcdd014353909edeea54ebd1e8241886f7abc9f715ca2f1025dac44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"0cf8bd41b605f3c023454134078e0b29fb24116c4ab84dc44eaed46533bbc5d6","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-79cvw2\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x23c456aecc90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-8lpbkjqgxtmb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-79cvw2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-79cvw2\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x23c456aecc90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-8lpbkjqgxtmb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-79cvw2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":22,"failure_pattern_ids":["07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610"],"matched_occurrences":22,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":4,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T23:00:45Z","started_at":"2026-10-03T23:00:45Z","completed_at":"2026-10-04T00:25:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-04T00:26:41.290696754Z","timing_metadata_checked_at":"2026-10-04T00:26:41.290696754Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-04T00:26:41.293780245Z","region_metadata_checked_at":"2026-10-04T00:26:41.293780245Z"},"failed_at":["e2e"],"failed_test_count":14,"occurrences":[{"row_id":"07615d8874a0cfc8bd62083e1b2a26b8abc1ce63b444a08aade6f3ff35a7a270","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"d2485400b9fb14ab84f4a84285253177801b00cc9d4149c5818d0c29ddef9be2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x3731739b0018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4h58pvg4tw2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1659164e-7252-4678-9399-950feff9f29d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1659164e-7252-4678-9399-950feff9f29d\",\n \"name\": \"1659164e-7252-4678-9399-950feff9f29d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:15:46.648351931Z\",\n \"endTime\": \"2026-10-03T23:34:50.305129886Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-hkqzd-6mfw6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-hkqzd-7x9wt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-hkqzd-6mfw6: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-hkqzd-7x9wt: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x3731739b0018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4h58pvg4tw2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1659164e-7252-4678-9399-950feff9f29d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1659164e-7252-4678-9399-950feff9f29d\",\n \"name\": \"1659164e-7252-4678-9399-950feff9f29d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:15:46.648351931Z\",\n \"endTime\": \"2026-10-03T23:34:50.305129886Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-hkqzd-6mfw6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-hkqzd-7x9wt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-hkqzd-6mfw6: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-hkqzd-7x9wt: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0b9e243d5211d809d600326d4c8f7918795a868d1acf0b91d9ee26f96d68d44a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"4e5c1277a0285514fbe13ffd0e118b84890287d84121f73af4e99275508596ce","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x242124721260\u003e: \n failed to create NodePool np-zb9djq: failed starting nodepool creation \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\" in resourcegroup=\"rg-nightly-4-20-r55xk7zmjr9c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-r55xk7zmjr9c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zb9djq/nodePools/np-zb9djq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x242124721260\u003e: \n failed to create NodePool np-zb9djq: failed starting nodepool creation \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\" in resourcegroup=\"rg-nightly-4-20-r55xk7zmjr9c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-r55xk7zmjr9c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zb9djq/nodePools/np-zb9djq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"0c2ff36c9fa9d1c2814df9873f715f5d26c0e9c3b78280e32e1adf0c60c9ec65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"80a54d00d33387405efc62cd65294145e6838630c336a95f7908f0dc39f21afd","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x184038744400\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tzvzfh\" in resourcegroup=\"rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tzvzfh/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x184038744400\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tzvzfh\" in resourcegroup=\"rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tzvzfh/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1a27268c2efba9efc20b39873ef90f27c7c0a87fd8db29be02e9ab410377bf8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"88bfecb223eed4d7950743d5a839fc1088854870e36ffeb70cfcf0e0e832d4ce","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x366855210620\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4c8kl482c2st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\",\n \"name\": \"37548aa4-a4e7-458d-b686-fb1201996386\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:15:28.253854505Z\",\n \"endTime\": \"2026-10-03T23:34:30.095088976Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: Creating: virtualmachine creating or updating\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x366855210620\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4c8kl482c2st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\",\n \"name\": \"37548aa4-a4e7-458d-b686-fb1201996386\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:15:28.253854505Z\",\n \"endTime\": \"2026-10-03T23:34:30.095088976Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: Creating: virtualmachine creating or updating\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"25160e6ee80916e150628d46923e30708eb79575039019904a641266cb9789ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"5c797d19c80475dc8cd016a2f12b5969e5f66674a64c327fd5b8176cbb909c6a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x31a06c9a5d00\u003e: \n failed to create NodePool np-dz68ws: failed waiting for nodepool=\"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\" in resourcegroup=\"rg-nightly-5-0-4t5m8n4czrvj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"name\": \"2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:35:55.566084896Z\",\n \"endTime\": \"2026-10-03T23:55:00.222357543Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: Creating: virtualmachine creating or updating\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x31a06c9a5d00\u003e: \n failed to create NodePool np-dz68ws: failed waiting for nodepool=\"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\" in resourcegroup=\"rg-nightly-5-0-4t5m8n4czrvj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"name\": \"2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:35:55.566084896Z\",\n \"endTime\": \"2026-10-03T23:55:00.222357543Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: Creating: virtualmachine creating or updating\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"311aed6905455f01a24ce18b915edf795a70b35896aaf94a63ce79bf1bbdb810","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"0c43de85fc2d6a7c071c55d1f96008747d85103a5e49f90377c2b68757383d44","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x36252fbe4930\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-548xlmq4vp84/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x36252fbe4930\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-548xlmq4vp84/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"33e5084d82fc1f34d4dc1b1eee6e3646f5610b03a122a91f38ceae0075e4eca8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"4220fe7603791b9ddfb09840fb8cd4a59017f25f10351d3998d45dda9b1f0d93","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3102b57c8300\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-5xj4fb5fc6dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3102b57c8300\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-5xj4fb5fc6dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"63a18ee70b6dd3fadee56e8db1f5f3815a03c5ddd42622454beb047252f63f42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"390cd3c454160d58c3f66648e63a7907c2041ab6f7cec42614cfb83642a9075a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34ef387fcea0\u003e: \n failed to create NodePool np-vgspf2: failed waiting for nodepool=\"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\" in resourcegroup=\"rg-nightly-4-21-dsfs76js9854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"name\": \"78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:34:31.068400483Z\",\n \"endTime\": \"2026-10-03T23:53:40.358231747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: Creating: virtualmachine creating or updating\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34ef387fcea0\u003e: \n failed to create NodePool np-vgspf2: failed waiting for nodepool=\"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\" in resourcegroup=\"rg-nightly-4-21-dsfs76js9854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"name\": \"78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:34:31.068400483Z\",\n \"endTime\": \"2026-10-03T23:53:40.358231747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: Creating: virtualmachine creating or updating\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8577416f451a2ed36ed114a6b22811e1bb91aa89f9fd3decda2a04485274719f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"e9efc6e08915eb27cce0e9b2aee48e6761e029a06f9f351f49af927ae38ed282","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4pbb46\" for cluster \"cluster-nightly-5-1-4pbb46\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1fb215239360\u003e: \n failed to create NodePool np-4pbb46: failed waiting for nodepool=\"np-4pbb46\" for cluster \"cluster-nightly-5-1-4pbb46\" in resourcegroup=\"rg-nightly-5-1-4rftrv2tqf7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\",\n \"name\": \"e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:37:12.955436587Z\",\n \"endTime\": \"2026-10-03T23:56:20.22170931Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-4s8vt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-rvt6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-4s8vt: Creating: virtualmachine creating or updating\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-rvt6q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4pbb46\" for cluster \"cluster-nightly-5-1-4pbb46\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1fb215239360\u003e: \n failed to create NodePool np-4pbb46: failed waiting for nodepool=\"np-4pbb46\" for cluster \"cluster-nightly-5-1-4pbb46\" in resourcegroup=\"rg-nightly-5-1-4rftrv2tqf7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\",\n \"name\": \"e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:37:12.955436587Z\",\n \"endTime\": \"2026-10-03T23:56:20.22170931Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-4s8vt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-rvt6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-4s8vt: Creating: virtualmachine creating or updating\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-rvt6q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"99de6c78e3de7de9d60851b5d6fbcd83dfbfb01b29445379aed449ddc179129f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"9a1223f045ad2c43a2d2b9f5c9d15450c0250574d5337dd1a988d4371b014517","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x2f86d9c91d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-4kkvfds7xktp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x2f86d9c91d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-4kkvfds7xktp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9e96f03b2056f80dcb253c5f798b3bf67b5e5d5b6323fdbc0266869acd8b507f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"e723e28bcd7e435dcc3421287fec024c49aa82e18f5dc42262402d55af5c17bd","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x145ed3760d50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-cxtlbfhqtdd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x145ed3760d50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-cxtlbfhqtdd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b946f60fd0e2b4c95b90842c71b4fb94c7565abfd4452b6a8c72d8cdd7d0bdca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"2b24f7ee1c45683523e52ab6a6fd28bf4368385c2a3884bc30ecbb5e5516b086","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x104897805d20\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r8db5d4ldfmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"name\": \"12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:41:40.611562032Z\",\n \"endTime\": \"2026-10-04T00:00:46.799966407Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: Creating: virtualmachine creating or updating\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x104897805d20\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r8db5d4ldfmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"name\": \"12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:41:40.611562032Z\",\n \"endTime\": \"2026-10-04T00:00:46.799966407Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: Creating: virtualmachine creating or updating\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management"}},{"row_id":"c8d4cbad50573632950a267892092074d3b0e567ad2d7352a039303c74fc38d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"1c86c022cf7d0c7808be0cd45d82052e35c88c4c35fb78fb4a2b9cf8b10f84d2","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x336e50b8a360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-njsbz992h4xx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x336e50b8a360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-njsbz992h4xx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d10b5b185889c09cae049e8dbd76615e7fac1711d9d0a2a267a45afb4178e88a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"f9761523403367d45961176e4cd187bb59f7a5f7a10bfd1b755e1c02f0042a38","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-hz7c47\" for cluster \"cluster-nightly-4-23-hz7c47\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33bfacf01ac0\u003e: \n failed to create NodePool np-hz7c47: failed waiting for nodepool=\"np-hz7c47\" for cluster \"cluster-nightly-4-23-hz7c47\" in resourcegroup=\"rg-nightly-4-23-zskzh2wkzsf8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ea1a45b2-8ea6-4372-9874-0ab41e3da185\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ea1a45b2-8ea6-4372-9874-0ab41e3da185\",\n \"name\": \"ea1a45b2-8ea6-4372-9874-0ab41e3da185\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:36:57.511630158Z\",\n \"endTime\": \"2026-10-03T23:56:00.189645328Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-hmv9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-qxrgm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-hmv9g: Creating: virtualmachine creating or updating\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-qxrgm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-hz7c47\" for cluster \"cluster-nightly-4-23-hz7c47\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33bfacf01ac0\u003e: \n failed to create NodePool np-hz7c47: failed waiting for nodepool=\"np-hz7c47\" for cluster \"cluster-nightly-4-23-hz7c47\" in resourcegroup=\"rg-nightly-4-23-zskzh2wkzsf8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ea1a45b2-8ea6-4372-9874-0ab41e3da185\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ea1a45b2-8ea6-4372-9874-0ab41e3da185\",\n \"name\": \"ea1a45b2-8ea6-4372-9874-0ab41e3da185\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:36:57.511630158Z\",\n \"endTime\": \"2026-10-03T23:56:00.189645328Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-hmv9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-qxrgm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-hmv9g: Creating: virtualmachine creating or updating\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-qxrgm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":14,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106215444435177472","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T02:51:08Z","started_at":"2026-10-03T02:51:08Z","completed_at":"2026-10-03T03:55:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T04:04:04.941200231Z","timing_metadata_checked_at":"2026-10-03T04:04:04.941200231Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-03T04:04:04.945480217Z","region_metadata_checked_at":"2026-10-03T05:07:22.822875223Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"05519656a6426c31fdaeeeffe3f1eba7fd19497f6c622f9b4fc2310fd9ee2272","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106215444435177472","occurred_at":"2026-10-03T02:51:08Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T02:00:44Z","started_at":"2026-10-03T02:00:44Z","completed_at":"2026-10-03T04:45:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T04:56:48.069203567Z","timing_metadata_checked_at":"2026-10-03T04:56:48.069203567Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T04:56:48.072716054Z","region_metadata_checked_at":"2026-10-03T04:56:48.072716054Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"1e82b028368cdac827fbf6dec9f74e79fdf0f46fc0a15e33fa8725c647c8b6f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"2c2caa46b0c9a860c8536cb1d0cfe4dc435fe9c87395f2009a2b50d36e765530","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1634a92060e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-5tmnm5zpfp6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2872059a-968a-400a-8256-03392979308f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2872059a-968a-400a-8256-03392979308f\",\n \"name\": \"2872059a-968a-400a-8256-03392979308f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:59:17.985754909Z\",\n \"endTime\": \"2026-10-03T03:18:20.080261329Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-x5mt5-q58ts: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-x5mt5-q58ts: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1634a92060e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-5tmnm5zpfp6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2872059a-968a-400a-8256-03392979308f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2872059a-968a-400a-8256-03392979308f\",\n \"name\": \"2872059a-968a-400a-8256-03392979308f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:59:17.985754909Z\",\n \"endTime\": \"2026-10-03T03:18:20.080261329Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-x5mt5-q58ts: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-x5mt5-q58ts: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3e28209d0365c5b58e497a8180b38441bb85bcf23cd1705add37bbf1a0158370","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"6ed585f3f050ad8b54d850a6ae0fdd9b4909a19d9c493dbc159c6d4911096dd3","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1c6f23908030\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ddfqq694gxhd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bea67035-4bd4-49aa-97d8-eb0a71fb82dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bea67035-4bd4-49aa-97d8-eb0a71fb82dc\",\n \"name\": \"bea67035-4bd4-49aa-97d8-eb0a71fb82dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:59:14.55416176Z\",\n \"endTime\": \"2026-10-03T03:18:19.785989613Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-7d6m6-8kqhg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-7d6m6-ptnv4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-7d6m6-8kqhg: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-7d6m6-ptnv4: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1c6f23908030\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ddfqq694gxhd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bea67035-4bd4-49aa-97d8-eb0a71fb82dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bea67035-4bd4-49aa-97d8-eb0a71fb82dc\",\n \"name\": \"bea67035-4bd4-49aa-97d8-eb0a71fb82dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:59:14.55416176Z\",\n \"endTime\": \"2026-10-03T03:18:19.785989613Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-7d6m6-8kqhg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-7d6m6-ptnv4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-7d6m6-8kqhg: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-7d6m6-ptnv4: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4e78cd9ac7b4f2b18f4b62e1706dd30bccb5e1ebd552ab8af8bec65fc5705015","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"e9fa9ae8e858a7b5355cb1b43d0d6779abb4d6d970a6d15f0440c7b03e009e78","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wgvn44\" for cluster \"cluster-candidate-5-1-wgvn44\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x582bdd8e0c0\u003e: \n failed to create NodePool np-wgvn44: failed waiting for nodepool=\"np-wgvn44\" for cluster \"cluster-candidate-5-1-wgvn44\" in resourcegroup=\"rg-candidate-5-1-g5g8pnhdz6gx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b96711e-b171-48d1-87d7-1f055025bc87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b96711e-b171-48d1-87d7-1f055025bc87\",\n \"name\": \"7b96711e-b171-48d1-87d7-1f055025bc87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:24:01.084946017Z\",\n \"endTime\": \"2026-10-03T03:43:03.38976371Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-5mgvm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-vkmcn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-5mgvm: Creating: virtualmachine creating or updating\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-vkmcn: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wgvn44\" for cluster \"cluster-candidate-5-1-wgvn44\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x582bdd8e0c0\u003e: \n failed to create NodePool np-wgvn44: failed waiting for nodepool=\"np-wgvn44\" for cluster \"cluster-candidate-5-1-wgvn44\" in resourcegroup=\"rg-candidate-5-1-g5g8pnhdz6gx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b96711e-b171-48d1-87d7-1f055025bc87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b96711e-b171-48d1-87d7-1f055025bc87\",\n \"name\": \"7b96711e-b171-48d1-87d7-1f055025bc87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:24:01.084946017Z\",\n \"endTime\": \"2026-10-03T03:43:03.38976371Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-5mgvm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-vkmcn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-5mgvm: Creating: virtualmachine creating or updating\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-vkmcn: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"533cc6932efa5876a88364efb445e7b106967255edac7ab9cc37002d995d4219","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"9528ad897be3c780f7f61e03b1ee8a6fe0b00f376a6ff34c98cd8ee75346769c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-994gtb\" for cluster \"cluster-candidate-5-0-994gtb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b59f6a200a0\u003e: \n failed to create NodePool np-994gtb: failed waiting for nodepool=\"np-994gtb\" for cluster \"cluster-candidate-5-0-994gtb\" in resourcegroup=\"rg-candidate-5-0-dpqs94628k6g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f3ae09fa-a5f4-44d2-9f00-68063627145c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f3ae09fa-a5f4-44d2-9f00-68063627145c\",\n \"name\": \"f3ae09fa-a5f4-44d2-9f00-68063627145c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:48.672584866Z\",\n \"endTime\": \"2026-10-03T03:41:53.397092955Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-9xspx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-v8vvg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-9xspx: Creating: virtualmachine creating or updating\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-v8vvg: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-994gtb\" for cluster \"cluster-candidate-5-0-994gtb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b59f6a200a0\u003e: \n failed to create NodePool np-994gtb: failed waiting for nodepool=\"np-994gtb\" for cluster \"cluster-candidate-5-0-994gtb\" in resourcegroup=\"rg-candidate-5-0-dpqs94628k6g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f3ae09fa-a5f4-44d2-9f00-68063627145c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f3ae09fa-a5f4-44d2-9f00-68063627145c\",\n \"name\": \"f3ae09fa-a5f4-44d2-9f00-68063627145c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:48.672584866Z\",\n \"endTime\": \"2026-10-03T03:41:53.397092955Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-9xspx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-v8vvg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-9xspx: Creating: virtualmachine creating or updating\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-v8vvg: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6f35cced70610dce46b749d6d7e1a72365d3fc153d4f4328bf225b9a12c8c996","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"fd808e83579b395ae5abd54c9de0aa37f2d5bc4ae6a53ba2e695ba0fa5c2dbc8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zsxxnl\" for cluster \"cluster-candidate-4-22-zsxxnl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x8479408b140\u003e: \n failed to create NodePool np-zsxxnl: failed waiting for nodepool=\"np-zsxxnl\" for cluster \"cluster-candidate-4-22-zsxxnl\" in resourcegroup=\"rg-candidate-4-22-7822rkz4qt8d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fd0879a-ebbb-4fea-b214-ece8494edab0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fd0879a-ebbb-4fea-b214-ece8494edab0\",\n \"name\": \"9fd0879a-ebbb-4fea-b214-ece8494edab0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:25.37619508Z\",\n \"endTime\": \"2026-10-03T03:41:30.020299253Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-6hmr2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-b6wrl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-6hmr2: Creating: virtualmachine creating or updating\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-b6wrl: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zsxxnl\" for cluster \"cluster-candidate-4-22-zsxxnl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x8479408b140\u003e: \n failed to create NodePool np-zsxxnl: failed waiting for nodepool=\"np-zsxxnl\" for cluster \"cluster-candidate-4-22-zsxxnl\" in resourcegroup=\"rg-candidate-4-22-7822rkz4qt8d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fd0879a-ebbb-4fea-b214-ece8494edab0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fd0879a-ebbb-4fea-b214-ece8494edab0\",\n \"name\": \"9fd0879a-ebbb-4fea-b214-ece8494edab0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:25.37619508Z\",\n \"endTime\": \"2026-10-03T03:41:30.020299253Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-6hmr2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-b6wrl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-6hmr2: Creating: virtualmachine creating or updating\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-b6wrl: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"812731e7ce1217c0a8cf1d47e29afbce21140f7de2697c517ff08150e4fc7e1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"882fba6cad22093956999ee2384f3228d45b1777a330537ec945b37008289a0a","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x933c5f12a60\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lbkmsp45kpxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\",\n \"name\": \"efe85861-3de4-4231-94f4-c87043439307\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:54:52.260057343Z\",\n \"endTime\": \"2026-10-03T04:13:53.608229594Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: Creating: virtualmachine creating or updating\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x933c5f12a60\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lbkmsp45kpxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\",\n \"name\": \"efe85861-3de4-4231-94f4-c87043439307\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:54:52.260057343Z\",\n \"endTime\": \"2026-10-03T04:13:53.608229594Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: Creating: virtualmachine creating or updating\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management"}},{"row_id":"8167bec06508c0a0cb844a4e4c1a98c880b64dfca1cb775ca53050adf2c6443d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"39751860e6d26d74a5f620ec93a78ea99d0e96ecd0ed7587dea64af7ba46a985","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xe7412ec40a8\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-j2962r94fd6x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"name\": \"4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:00:19.581988952Z\",\n \"endTime\": \"2026-10-03T03:19:19.770602537Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xe7412ec40a8\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-j2962r94fd6x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"name\": \"4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:00:19.581988952Z\",\n \"endTime\": \"2026-10-03T03:19:19.770602537Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"986d7547b214789d5aea01f33a08eddf88dfb49f80123dcaa2ff9b05368cefa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"8ed18e43fabdf08315daac2d8e02c94f2c8a172b70d38853c874dbc955f5d982","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x21c259b32dc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-2sgbstf7nlxj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3898db04-618a-4491-92e8-a44483e69093\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3898db04-618a-4491-92e8-a44483e69093\",\n \"name\": \"3898db04-618a-4491-92e8-a44483e69093\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:58:32.753298506Z\",\n \"endTime\": \"2026-10-03T03:17:39.788365749Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-hbt2g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-zpxs2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-hbt2g: Creating: virtualmachine creating or updating\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-zpxs2: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x21c259b32dc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-2sgbstf7nlxj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3898db04-618a-4491-92e8-a44483e69093\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3898db04-618a-4491-92e8-a44483e69093\",\n \"name\": \"3898db04-618a-4491-92e8-a44483e69093\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:58:32.753298506Z\",\n \"endTime\": \"2026-10-03T03:17:39.788365749Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-hbt2g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-zpxs2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-hbt2g: Creating: virtualmachine creating or updating\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-zpxs2: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a64bfbf378aa9bc2d91396a3c062810db32dc76603cadbc28cb269b1b69e317d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"8dc98ddc487d792e30d36e88cd34b10e8b904d76ef343c1f573a64f712f3b220","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x15f1e13c9ee0\u003e: \n failed to create NodePool np-6tqvsp: failed waiting for nodepool=\"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\" in resourcegroup=\"rg-candidate-4-21-qxp54ksqdkvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"name\": \"cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:23:37.870437581Z\",\n \"endTime\": \"2026-10-03T03:42:43.397028044Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: Creating: virtualmachine creating or updating\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x15f1e13c9ee0\u003e: \n failed to create NodePool np-6tqvsp: failed waiting for nodepool=\"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\" in resourcegroup=\"rg-candidate-4-21-qxp54ksqdkvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"name\": \"cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:23:37.870437581Z\",\n \"endTime\": \"2026-10-03T03:42:43.397028044Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: Creating: virtualmachine creating or updating\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cd8ada755a7b03b78d2ecbcd9dbb6d9a148fa6a6a8f57a8233b64c5a9f879eac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"e5c35e2b9b861c774ee1789f184fbf1cb2f5f308d8dc05c180346b85e646fe05","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vlg9bq\" for cluster \"cluster-candidate-4-23-vlg9bq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e6e62f6cda0\u003e: \n failed to create NodePool np-vlg9bq: failed waiting for nodepool=\"np-vlg9bq\" for cluster \"cluster-candidate-4-23-vlg9bq\" in resourcegroup=\"rg-candidate-4-23-dfclsfklgxtx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98169a27-755e-48b1-800f-32bc5e94d96c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98169a27-755e-48b1-800f-32bc5e94d96c\",\n \"name\": \"98169a27-755e-48b1-800f-32bc5e94d96c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:35.181930151Z\",\n \"endTime\": \"2026-10-03T03:41:39.859849458Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-mqggg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-zfwnx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-mqggg: Creating: virtualmachine creating or updating\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-zfwnx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vlg9bq\" for cluster \"cluster-candidate-4-23-vlg9bq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e6e62f6cda0\u003e: \n failed to create NodePool np-vlg9bq: failed waiting for nodepool=\"np-vlg9bq\" for cluster \"cluster-candidate-4-23-vlg9bq\" in resourcegroup=\"rg-candidate-4-23-dfclsfklgxtx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98169a27-755e-48b1-800f-32bc5e94d96c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98169a27-755e-48b1-800f-32bc5e94d96c\",\n \"name\": \"98169a27-755e-48b1-800f-32bc5e94d96c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:35.181930151Z\",\n \"endTime\": \"2026-10-03T03:41:39.859849458Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-mqggg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-zfwnx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-mqggg: Creating: virtualmachine creating or updating\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-zfwnx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e626ee7d005666c5d5f396c38b3aef1b15642cd86798c690de5419fc7fa8f199","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"2c9f39e0ab026a6227defad21130e9e9d394ff6a055a7d1fa925b7fc0262141f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7k8q7d\" for cluster \"cluster-candidate-4-20-7k8q7d\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x342eb43e6c40\u003e: \n failed to create NodePool np-7k8q7d: failed waiting for nodepool=\"np-7k8q7d\" for cluster \"cluster-candidate-4-20-7k8q7d\" in resourcegroup=\"rg-candidate-4-20-vn2gzb9bgx68\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f5da487-7370-42cc-8931-634a89f3149c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f5da487-7370-42cc-8931-634a89f3149c\",\n \"name\": \"8f5da487-7370-42cc-8931-634a89f3149c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:18:20.263677419Z\",\n \"endTime\": \"2026-10-03T03:37:30.013101747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-rlpsx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-v5gzq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-rlpsx: Creating: virtualmachine creating or updating\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-v5gzq: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7k8q7d\" for cluster \"cluster-candidate-4-20-7k8q7d\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x342eb43e6c40\u003e: \n failed to create NodePool np-7k8q7d: failed waiting for nodepool=\"np-7k8q7d\" for cluster \"cluster-candidate-4-20-7k8q7d\" in resourcegroup=\"rg-candidate-4-20-vn2gzb9bgx68\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f5da487-7370-42cc-8931-634a89f3149c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f5da487-7370-42cc-8931-634a89f3149c\",\n \"name\": \"8f5da487-7370-42cc-8931-634a89f3149c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:18:20.263677419Z\",\n \"endTime\": \"2026-10-03T03:37:30.013101747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-rlpsx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-v5gzq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-rlpsx: Creating: virtualmachine creating or updating\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-v5gzq: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T00:25:08Z","started_at":"2026-10-03T00:25:08Z","completed_at":"2026-10-03T02:43:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T03:00:49.285108674Z","timing_metadata_checked_at":"2026-10-03T03:00:49.285108674Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T03:00:49.288123164Z","region_metadata_checked_at":"2026-10-03T03:00:49.288123164Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"32dd870e25444edc34f0a54c4f5de17f756f32c1abedbbc2dff3c7f667d5bc09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"d6f5de42eefb7152643e950681559e711013d084e28baf1d882e6f029c0e5511","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xca0cd836f80\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"name\": \"4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:51:49.926221366Z\",\n \"endTime\": \"2026-10-03T02:10:56.26372719Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: Creating: virtualmachine creating or updating\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xca0cd836f80\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"name\": \"4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:51:49.926221366Z\",\n \"endTime\": \"2026-10-03T02:10:56.26372719Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: Creating: virtualmachine creating or updating\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management"}},{"row_id":"46b26e75380dd03099914ca7e74dae6badab952d4d44aae6a15e8e2881d52082","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"475946b6879259ac2394beb241c800f706ea9c7b18c2b17cfb61377bbdc2cf11","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1363e14cc018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-jtmvgv9m6829\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"name\": \"31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:01:05.764058109Z\",\n \"endTime\": \"2026-10-03T01:20:12.852653102Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1363e14cc018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-jtmvgv9m6829\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"name\": \"31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:01:05.764058109Z\",\n \"endTime\": \"2026-10-03T01:20:12.852653102Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"55886caa49c7a303baada857ec16c42b3ce5bb633b0bed38f3bd2d35c9cb2f49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"2270ab4d2942cd093aff353fd34037a69c93e335ec7f1a88dfa214a3f4004534","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7m65d4\" for cluster \"cluster-candidate-4-21-7m65d4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xdd74a91040\u003e: \n failed to create NodePool np-7m65d4: failed waiting for nodepool=\"np-7m65d4\" for cluster \"cluster-candidate-4-21-7m65d4\" in resourcegroup=\"rg-candidate-4-21-cdkldcpn7c4w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f45b9d58-6bb8-408f-943e-164ec6193300\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f45b9d58-6bb8-408f-943e-164ec6193300\",\n \"name\": \"f45b9d58-6bb8-408f-943e-164ec6193300\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:23:06.848197915Z\",\n \"endTime\": \"2026-10-03T01:42:11.810063991Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-7jtvb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-8fd6d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-7jtvb: Creating: virtualmachine creating or updating\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-8fd6d: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7m65d4\" for cluster \"cluster-candidate-4-21-7m65d4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xdd74a91040\u003e: \n failed to create NodePool np-7m65d4: failed waiting for nodepool=\"np-7m65d4\" for cluster \"cluster-candidate-4-21-7m65d4\" in resourcegroup=\"rg-candidate-4-21-cdkldcpn7c4w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f45b9d58-6bb8-408f-943e-164ec6193300\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f45b9d58-6bb8-408f-943e-164ec6193300\",\n \"name\": \"f45b9d58-6bb8-408f-943e-164ec6193300\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:23:06.848197915Z\",\n \"endTime\": \"2026-10-03T01:42:11.810063991Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-7jtvb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-8fd6d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-7jtvb: Creating: virtualmachine creating or updating\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-8fd6d: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"713dd46977abda13d0ec8b75a9ad33bbcccef6aff09092181e72e6d01b9456d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"8f5972671bc46aef70b454f5d37eb1f821ffe511b2716373992fdcfdd6e793dd","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x356eae117080\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-2xqzzh8h7jx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2692539-c900-4d9b-9952-7a76ad7fc6a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2692539-c900-4d9b-9952-7a76ad7fc6a0\",\n \"name\": \"c2692539-c900-4d9b-9952-7a76ad7fc6a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:56:20.964200965Z\",\n \"endTime\": \"2026-10-03T01:15:22.786636188Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-kbv8l-5qvk6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-kbv8l-5qvk6: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x356eae117080\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-2xqzzh8h7jx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2692539-c900-4d9b-9952-7a76ad7fc6a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2692539-c900-4d9b-9952-7a76ad7fc6a0\",\n \"name\": \"c2692539-c900-4d9b-9952-7a76ad7fc6a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:56:20.964200965Z\",\n \"endTime\": \"2026-10-03T01:15:22.786636188Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-kbv8l-5qvk6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-kbv8l-5qvk6: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7970d9eef7bcdbc5fdd8d98ccb5de27654e4a45317dcb366505f419f46508f5e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"2a69c285def7ab9febd458730c74909c9a16be32934cbdbd5ca44844262b1eb0","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x36891cc75140\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-rw4pgrx2qsqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a719f074-9cbb-4f60-8f74-9acd323de195\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a719f074-9cbb-4f60-8f74-9acd323de195\",\n \"name\": \"a719f074-9cbb-4f60-8f74-9acd323de195\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:58:50.620270926Z\",\n \"endTime\": \"2026-10-03T01:17:53.07846241Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-nmjht: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-qnrnm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-nmjht: Creating: virtualmachine creating or updating\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-qnrnm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x36891cc75140\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-rw4pgrx2qsqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a719f074-9cbb-4f60-8f74-9acd323de195\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a719f074-9cbb-4f60-8f74-9acd323de195\",\n \"name\": \"a719f074-9cbb-4f60-8f74-9acd323de195\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:58:50.620270926Z\",\n \"endTime\": \"2026-10-03T01:17:53.07846241Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-nmjht: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-qnrnm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-nmjht: Creating: virtualmachine creating or updating\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-qnrnm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"89d87ec14c3e09c46fef3fce7a4099ca7d20f514ce909b2e541481cf32e92feb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"06c296f07902755728beb3838eb1ec4ac170f38778b421b87c5fc204e64d1fe4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7bgsbj\" for cluster \"cluster-candidate-5-1-7bgsbj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x355ed6679960\u003e: \n failed to create NodePool np-7bgsbj: failed waiting for nodepool=\"np-7bgsbj\" for cluster \"cluster-candidate-5-1-7bgsbj\" in resourcegroup=\"rg-candidate-5-1-twzxzjn6gcp4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48ddacde-704d-413e-818f-fb753e5b6f29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48ddacde-704d-413e-818f-fb753e5b6f29\",\n \"name\": \"48ddacde-704d-413e-818f-fb753e5b6f29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:21:40.063888823Z\",\n \"endTime\": \"2026-10-03T01:40:41.803400223Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-b2m7f: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-ssxpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-b2m7f: Creating: virtualmachine creating or updating\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-ssxpc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7bgsbj\" for cluster \"cluster-candidate-5-1-7bgsbj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x355ed6679960\u003e: \n failed to create NodePool np-7bgsbj: failed waiting for nodepool=\"np-7bgsbj\" for cluster \"cluster-candidate-5-1-7bgsbj\" in resourcegroup=\"rg-candidate-5-1-twzxzjn6gcp4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48ddacde-704d-413e-818f-fb753e5b6f29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48ddacde-704d-413e-818f-fb753e5b6f29\",\n \"name\": \"48ddacde-704d-413e-818f-fb753e5b6f29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:21:40.063888823Z\",\n \"endTime\": \"2026-10-03T01:40:41.803400223Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-b2m7f: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-ssxpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-b2m7f: Creating: virtualmachine creating or updating\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-ssxpc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8b0ebde71eb2f691daf70a2bcd3cc4703f41ec171e90b55dfcf6dfefbcc160bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"ed4c47b0094dced5127989ca5fda7f6a4352c3e90ee689cf14e5def8764d9f1d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x382d24c28030\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-zvljfb77qpnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd6903c4-bc0c-4634-b456-f0053f4e62ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd6903c4-bc0c-4634-b456-f0053f4e62ce\",\n \"name\": \"fd6903c4-bc0c-4634-b456-f0053f4e62ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:59:54.046743948Z\",\n \"endTime\": \"2026-10-03T01:19:03.00985984Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-2hhfv-7r58z: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-2hhfv-fmwpb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-2hhfv-7r58z: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-2hhfv-fmwpb: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x382d24c28030\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-zvljfb77qpnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd6903c4-bc0c-4634-b456-f0053f4e62ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd6903c4-bc0c-4634-b456-f0053f4e62ce\",\n \"name\": \"fd6903c4-bc0c-4634-b456-f0053f4e62ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:59:54.046743948Z\",\n \"endTime\": \"2026-10-03T01:19:03.00985984Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-2hhfv-7r58z: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-2hhfv-fmwpb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-2hhfv-7r58z: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-2hhfv-fmwpb: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a90f150f6c71ea9e51ea7d96e09f6f2f22abc4c9d527371c73769d9f53c0f0a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"173a7e9edaf6f61e13589f056617df6ddb63fa543ce96dbe851cba6372717a02","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-mpbw9s\" for cluster \"cluster-candidate-5-0-mpbw9s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xea20e6b7700\u003e: \n failed to create NodePool np-mpbw9s: failed waiting for nodepool=\"np-mpbw9s\" for cluster \"cluster-candidate-5-0-mpbw9s\" in resourcegroup=\"rg-candidate-5-0-6l9pw288lpdv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f8bd06-343a-4b31-8a4d-21de9398ca35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f8bd06-343a-4b31-8a4d-21de9398ca35\",\n \"name\": \"49f8bd06-343a-4b31-8a4d-21de9398ca35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:23:58.4098212Z\",\n \"endTime\": \"2026-10-03T01:43:01.804835963Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-sx8g2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-xvdjz: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-sx8g2: Creating: virtualmachine creating or updating\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-xvdjz: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-mpbw9s\" for cluster \"cluster-candidate-5-0-mpbw9s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xea20e6b7700\u003e: \n failed to create NodePool np-mpbw9s: failed waiting for nodepool=\"np-mpbw9s\" for cluster \"cluster-candidate-5-0-mpbw9s\" in resourcegroup=\"rg-candidate-5-0-6l9pw288lpdv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f8bd06-343a-4b31-8a4d-21de9398ca35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f8bd06-343a-4b31-8a4d-21de9398ca35\",\n \"name\": \"49f8bd06-343a-4b31-8a4d-21de9398ca35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:23:58.4098212Z\",\n \"endTime\": \"2026-10-03T01:43:01.804835963Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-sx8g2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-xvdjz: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-sx8g2: Creating: virtualmachine creating or updating\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-xvdjz: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"aa6e65fd1b264e949cc75090b7fc457a827a9e08168675f5958a989ac9458eb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"34a4dddd4ac406094ca0f1a4bf7eb674580835c68e05a723aea1b94befa47471","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-55sh69\" for cluster \"cluster-candidate-4-23-55sh69\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d83ca910b00\u003e: \n failed to create NodePool np-55sh69: failed waiting for nodepool=\"np-55sh69\" for cluster \"cluster-candidate-4-23-55sh69\" in resourcegroup=\"rg-candidate-4-23-b2jh9rr7848q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/857aaaff-55f2-4507-bff3-f5f99ad82196\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/857aaaff-55f2-4507-bff3-f5f99ad82196\",\n \"name\": \"857aaaff-55f2-4507-bff3-f5f99ad82196\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:21:00.037991541Z\",\n \"endTime\": \"2026-10-03T01:40:02.219519171Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-qt7pp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-xbwxp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-qt7pp: Creating: virtualmachine creating or updating\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-xbwxp: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-55sh69\" for cluster \"cluster-candidate-4-23-55sh69\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d83ca910b00\u003e: \n failed to create NodePool np-55sh69: failed waiting for nodepool=\"np-55sh69\" for cluster \"cluster-candidate-4-23-55sh69\" in resourcegroup=\"rg-candidate-4-23-b2jh9rr7848q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/857aaaff-55f2-4507-bff3-f5f99ad82196\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/857aaaff-55f2-4507-bff3-f5f99ad82196\",\n \"name\": \"857aaaff-55f2-4507-bff3-f5f99ad82196\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:21:00.037991541Z\",\n \"endTime\": \"2026-10-03T01:40:02.219519171Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-qt7pp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-xbwxp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-qt7pp: Creating: virtualmachine creating or updating\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-xbwxp: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b6acb3809f5fabcdab967ed5a2340883e9c3f3707e6dd000e57e721aff58222d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"38fdd980bf52332969e2e4f977878b2b3caffd7b67413af3dedd10df203bfc1a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xcs2zx\" for cluster \"cluster-candidate-4-20-xcs2zx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bb65f4dc3e0\u003e: \n failed to create NodePool np-xcs2zx: failed waiting for nodepool=\"np-xcs2zx\" for cluster \"cluster-candidate-4-20-xcs2zx\" in resourcegroup=\"rg-candidate-4-20-9tm8m64jwh6q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5e3d740-6442-4b8e-9013-9867a8b30bc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5e3d740-6442-4b8e-9013-9867a8b30bc0\",\n \"name\": \"b5e3d740-6442-4b8e-9013-9867a8b30bc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:20:28.820955232Z\",\n \"endTime\": \"2026-10-03T01:39:31.794768159Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-42gm5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-zmvl5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-42gm5: Creating: virtualmachine creating or updating\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-zmvl5: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xcs2zx\" for cluster \"cluster-candidate-4-20-xcs2zx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bb65f4dc3e0\u003e: \n failed to create NodePool np-xcs2zx: failed waiting for nodepool=\"np-xcs2zx\" for cluster \"cluster-candidate-4-20-xcs2zx\" in resourcegroup=\"rg-candidate-4-20-9tm8m64jwh6q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5e3d740-6442-4b8e-9013-9867a8b30bc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5e3d740-6442-4b8e-9013-9867a8b30bc0\",\n \"name\": \"b5e3d740-6442-4b8e-9013-9867a8b30bc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:20:28.820955232Z\",\n \"endTime\": \"2026-10-03T01:39:31.794768159Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-42gm5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-zmvl5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-42gm5: Creating: virtualmachine creating or updating\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-zmvl5: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c414687d08f84faa025d931b3e0814969e9ebb6afca24f6d5dab4384b1ca5a07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"399671fb721e085227d1bcef5acf1cf7744be28e89d36f981b42cc7c1ec8a4d7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-md77w4\" for cluster \"cluster-candidate-4-22-md77w4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34d9aee24b60\u003e: \n failed to create NodePool np-md77w4: failed waiting for nodepool=\"np-md77w4\" for cluster \"cluster-candidate-4-22-md77w4\" in resourcegroup=\"rg-candidate-4-22-qt94z5jzjgsz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7b70034-c32d-4689-8c16-d6b19ed89cc8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7b70034-c32d-4689-8c16-d6b19ed89cc8\",\n \"name\": \"c7b70034-c32d-4689-8c16-d6b19ed89cc8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:25:32.783100892Z\",\n \"endTime\": \"2026-10-03T01:44:41.785015844Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-488zq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-f2khf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-488zq: Creating: virtualmachine creating or updating\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-f2khf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-md77w4\" for cluster \"cluster-candidate-4-22-md77w4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34d9aee24b60\u003e: \n failed to create NodePool np-md77w4: failed waiting for nodepool=\"np-md77w4\" for cluster \"cluster-candidate-4-22-md77w4\" in resourcegroup=\"rg-candidate-4-22-qt94z5jzjgsz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7b70034-c32d-4689-8c16-d6b19ed89cc8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7b70034-c32d-4689-8c16-d6b19ed89cc8\",\n \"name\": \"c7b70034-c32d-4689-8c16-d6b19ed89cc8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:25:32.783100892Z\",\n \"endTime\": \"2026-10-03T01:44:41.785015844Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-488zq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-f2khf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-488zq: Creating: virtualmachine creating or updating\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-f2khf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f30cc3ab26e40e608530edfdf1bd27479dccbc04d61f493b4ffee7dc3136f009","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"d5c37afd72eeef3b1717f0ed626a3a3f20e451ad1fbcbdcf4ca6127c60afb433","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xbbf668ab320\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lqwbkg\" in resourcegroup=\"rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"name\": \"a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:58:23.770970582Z\",\n \"endTime\": \"2026-10-03T02:17:26.319658092Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xbbf668ab320\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lqwbkg\" in resourcegroup=\"rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"name\": \"a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:58:23.770970582Z\",\n \"endTime\": \"2026-10-03T02:17:26.319658092Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}