{"meta":{"start_date":"2026-10-03","end_date":"2026-10-03","start_at":"2026-10-03T00:00:00Z","end_at":"2026-10-04T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T07:59:05Z","environments":["dev","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":39,"failed_runs":27,"raw_occurrences":40,"matched_occurrences":39,"runs_affected":26},"rows":[{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":15,"runs_affected":15,"failed_runs":15,"run_impact_percent":38.46153846153846,"anchor_occurrences":15,"after_last_push_count":38,"trend_counts":[2,7,11,3,5,19,15],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":99,"prior_last_seen_at":"2026-09-27T13:27:59Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":15}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T22:09:35Z\n Ended: 2026-10-03T22:15:33Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0690944-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T21:07:34Z\n Ended: 2026-10-03T21:13:37Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1818624-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T19:53:25Z\n Ended: 2026-10-03T19:59:26Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7836160-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106489060980690944","occurred_at":"2026-10-03T20:58:23Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7336,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106473224001818624","occurred_at":"2026-10-03T19:55:27Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7331,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106455066687836160","occurred_at":"2026-10-03T18:43:18Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7338,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428464000143360","occurred_at":"2026-10-03T16:57:35Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7331,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402405829578752","occurred_at":"2026-10-03T15:14:03Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7340,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106344080505049088","occurred_at":"2026-10-03T11:22:17Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7338,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106319269334618112","occurred_at":"2026-10-03T09:43:41Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7318,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7334/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106297012751175680","occurred_at":"2026-10-03T08:15:15Z","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","pr_number":7334,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106283184227356672","occurred_at":"2026-10-03T07:20:18Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7318,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232246682783744","occurred_at":"2026-10-03T03:57:54Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7336,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106223756383883264","occurred_at":"2026-10-03T03:24:09Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7332,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106216516587360256","occurred_at":"2026-10-03T02:55:23Z","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","pr_number":7333,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106201793171034112","occurred_at":"2026-10-03T01:56:53Z","signature_id":"fa412034903709961f37b7b9179b2905b4c4fb3d1b051e1833d51c25494e7359","pr_number":7331,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106186045094629376","occurred_at":"2026-10-03T00:54:18Z","signature_id":"fa412034903709961f37b7b9179b2905b4c4fb3d1b051e1833d51c25494e7359","pr_number":7335,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106184864796512256","occurred_at":"2026-10-03T00:49:37Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7336,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":10,"runs_affected":9,"failed_runs":9,"run_impact_percent":23.076923076923077,"anchor_occurrences":10,"after_last_push_count":37,"also_in":["PROD","STG"],"trend_counts":[7,19,10,8,21,21,10],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":91,"prior_last_seen_at":"2026-09-27T22:20:12Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f72d213c320\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-n6bsrg\" in resourcegroup=\"rg-np-noedge-n6bsrg-vxj5p9qrl42z\" to finish creating: GET https://rp.j2350080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\",\n \"name\": \"83d90b9f-3285-4912-9884-40a78690862b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-04T00:37:10.755457782Z\",\n \"endTime\": \"2026-10-04T00:56:17.299103912Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0x2f72d213c320\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-n6bsrg\" in resourcegroup=\"rg-np-noedge-n6bsrg-vxj5p9qrl42z\" to finish creating: GET https://rp.j2350080.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/83d90b9f-3285-4912-9884-40a78690862b\",\n \"name\": \"83d90b9f-3285-4912-9884-40a78690862b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-04T00:37:10.755457782Z\",\n \"endTime\": \"2026-10-04T00:56:17.299103912Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-2kn7g to exist\\nMachine i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-n6bsrg-vxj5p9qrl42z-np-ne-n6bsrg--managed/providers/Microsoft.Compute/virtualMachines/i4d7j9t8e0g6i1k-npnoedge-4-20-fgjsm-nh6rw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1590b4a34e00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-hd5zgd7fw6ql\" to finish creating: GET https://rp.j0239360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"name\": \"0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T18:22:53.322060806Z\",\n \"endTime\": \"2026-10-03T18:41:55.898243278Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkgjk-bhjgr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkgjk-bhjgr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1590b4a34e00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-hd5zgd7fw6ql\" to finish creating: GET https://rp.j0239360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"name\": \"0504a146-7d92-49ac-9646-6e1b202d7d94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T18:22:53.322060806Z\",\n \"endTime\": \"2026-10-03T18:41:55.898243278Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-bkgjk-bhjgr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-bkgjk-bhjgr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-hd5zgd7fw6ql--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-bkgjk-bhjgr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x1737a1124260\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kjgdvvpr4g6r\" to finish creating: GET https://rp.j0472448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"name\": \"9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T16:34:46.276618892Z\",\n \"endTime\": \"2026-10-03T16:53:53.086174722Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x1737a1124260\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kjgdvvpr4g6r\" to finish creating: GET https://rp.j0472448.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"name\": \"9760730d-edb7-41c2-8626-62f85b63fc1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T16:34:46.276618892Z\",\n \"endTime\": \"2026-10-03T16:53:53.086174722Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kjgdvvpr4g6r--managed/providers/Microsoft.Compute/virtualMachines/n0m4o0c1m0o6o8n-np-1-zf8dz-6rq54 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106522529362350080","occurred_at":"2026-10-03T23:11:22Z","signature_id":"501495b0aef70c6325fa26b9f0b745d7b4ce463804d55b753046dc2f9f5ed6e4","pr_number":7332,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428486020239360","occurred_at":"2026-10-03T16:57:41Z","signature_id":"09d3516d5937c201911e513dc777964c08f8d8e605d4d938d29eb1e35d21ff64","pr_number":7336,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402589850472448","occurred_at":"2026-10-03T15:14:47Z","signature_id":"afc72a53fa7774ed1734007a58442b2c5acbb5d61e6acaf495185cd900df7590","pr_number":7338,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106361385184137216","occurred_at":"2026-10-03T12:31:03Z","signature_id":"c05cf9daf6680f1855b02943b461dc08c3799199d299289700c2fc278277b2c4","pr_number":7318,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106344080505049088","occurred_at":"2026-10-03T11:22:17Z","signature_id":"21861e38aab43579ee30401f2af52d615d1c92a4943c93ec549c760780f28b16","pr_number":7338,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7338/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106344080505049088","occurred_at":"2026-10-03T11:22:17Z","signature_id":"3c3f2ccc36c35f1c9e21f03f7eff4418f1739e7896a5865697c85dc9064d117f","pr_number":7338,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7268/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106316215747088384","occurred_at":"2026-10-03T09:31:33Z","signature_id":"8e852f466cf78bfd32f7a3098547e4a2279ba62592d4769dc239478e539ec985","pr_number":7268,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232246682783744","occurred_at":"2026-10-03T03:57:54Z","signature_id":"4ecba326214d91b223bb8a28fa503efb6110b5f309f19014e5eec742eee54784","pr_number":7336,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106232206048366592","occurred_at":"2026-10-03T03:57:44Z","signature_id":"4cb0132308929f56cf10c7cf8dcc82eb33338d500ae0ff69d7f2b06b20772f9b","pr_number":7335,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106216516587360256","occurred_at":"2026-10-03T02:55:23Z","signature_id":"405132e552ca41557cf8c6f086fdcc0c25a7a484630f63163bb4bed7e7d238ad","pr_number":7333,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.128205128205129,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":20,"prior_last_seen_at":"2026-09-25T20:36:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3263781c03a0\u003e: \n DNS for route host agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3263781c03a0\u003e: \n DNS for route host agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-j98xq.apps.aro.v20261001.9je8.j4629376.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-xt2c7s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b2c75ad8a80\u003e: \n DNS for route host agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-xt2c7s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1b2c75ad8a80\u003e: \n DNS for route host agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-c8pn7.apps.aro.s8a7u2c4b6z3h8o.rc93.j6512256.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7335/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106186045094629376","occurred_at":"2026-10-03T00:54:18Z","signature_id":"5bfc63de429111a26e0e3c0120b6821322754898f203dabd0929d69d6732c4f1","pr_number":7335,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106184864796512256","occurred_at":"2026-10-03T00:49:37Z","signature_id":"12b342095018d1a5c43a8d0f0915ddf5043eaa8b66110e00ae45d0793de2806b","pr_number":7336,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.128205128205129,"anchor_occurrences":2,"after_last_push_count":65,"trend_counts":[2,1,7,10,6,5,2],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":134,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":2}],"full_error_samples":["alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-file-csi-driver-operator-56bd98ff47-n9mw7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-file-csi-driver-operator-56bd98ff47-n9mw7 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-disk-csi-driver-operator-68446dbd4d-gp5r7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-disk-csi-driver-operator-68446dbd4d-gp5r7 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-disk-csi-driver-operator-68446dbd4d-72x46\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-disk-csi-driver-operator-68446dbd4d-72x46 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-disk-csi-driver-operator-68446dbd4d-l8w9v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-disk-csi-driver-operator-68446dbd4d-l8w9v has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-03T18:23:01Z\n Ended: 2026-10-03T18:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0143360-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e\", pod=\"azure-file-csi-driver-operator-56bd98ff47-lhdbd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7fcgvvu6q2gat5neqdkaq9suafasab-y5s6m0p7w4t3q7e/azure-file-csi-driver-operator-56bd98ff47-lhdbd has been in a non-ready state for longer than 5 minutes.","alert fired 69 time(s) (63 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-52l2j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-52l2j has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-vdz8t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-vdz8t has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-k2787\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-k2787 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-zp9nc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-zp9nc has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-vk8sg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-vk8sg has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-03T17:36:48Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-fvs76\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-fvs76 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-llstd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-llstd has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-5mcng\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-5mcng has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9sqdg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9sqdg has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-03T17:36:57Z\n Ended: 2026-10-03T17:51:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-btfk4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-btfk4 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-dq2vz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-dq2vz has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-x6wz7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-x6wz7 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-ctpt9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-ctpt9 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-j2d9z\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-j2d9z has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-4h79j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-4h79j has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-z2w4b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-z2w4b has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-nkt4w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-nkt4w has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-ffhbr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-ffhbr has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-csvsv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-csvsv has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-s4h6v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-s4h6v has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-dl5l4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-dl5l4 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-47862\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-47862 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-gw8rm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-gw8rm has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-mrcs5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-mrcs5 has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-tzs9x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-tzs9x has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-pjk6c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-pjk6c has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-vdtmc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-vdtmc has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-7kmn4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-7kmn4 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-6w4zc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-6w4zc has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-mnfxd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-mnfxd has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-m84qg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-m84qg has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-j8kmn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-j8kmn has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-r827x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-r827x has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lscgk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lscgk has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-gj4v6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-gj4v6 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-2mvqm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-2mvqm has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lw6l6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lw6l6 has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9w425\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9w425 has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-ndzg5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-ndzg5 has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-b662v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-b662v has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-wj72x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-wj72x has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-5c5cd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-5c5cd has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lkvbz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lkvbz has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-22klt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-22klt has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-rxm4b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-rxm4b has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-r7qwh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-r7qwh has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-xhl5w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-xhl5w has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-h4p6b\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-h4p6b has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-j7f54\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-j7f54 has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-bs5j7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-bs5j7 has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-lwcng\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-lwcng has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9n44n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9n44n has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-dp4wn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-dp4wn has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-8qlz4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-8qlz4 has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-9n2td\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-9n2td has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-d6x6h\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-d6x6h has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-10-03T17:37:51Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-92ktd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-92ktd has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-sxxzw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-sxxzw has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:51:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-zvv2v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-zvv2v has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-2hsfs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-2hsfs has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-nz8k8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-nz8k8 has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-gssl7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-gssl7 has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-10-03T17:37:52Z\n Ended: 2026-10-03T17:56:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1091968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u\", pod=\"kube-controller-manager-5b899d5dd9-7865d\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t7en02emtg8ui89m978mkg1haofcv21-x6l3l6y2c5m1t6u/kube-controller-manager-5b899d5dd9-7865d has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106428464000143360","occurred_at":"2026-10-03T16:57:35Z","signature_id":"fbbc2741d1268c4182e25737f2913295ff9bf11b2acf044ba433a4c9be14b821","pr_number":7331,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106422169931091968","occurred_at":"2026-10-03T16:32:35Z","signature_id":"d03789470daaa6e20f0e10af6ab181b291bf54e6bdce46582adbae8892beb283","pr_number":7333,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.128205128205129,"anchor_occurrences":2,"after_last_push_count":10,"trend_counts":[4,7,1,0,0,3,2],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":34,"prior_last_seen_at":"2026-09-27T21:19:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106223756383883264","occurred_at":"2026-10-03T03:24:09Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7332,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7336/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106184864796512256","occurred_at":"2026-10-03T00:49:37Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":7336,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,2,0,0,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-24T09:33:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-vzb878crbjw5\" to finish creating: GET https://rp.j0614912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"name\": \"737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T14:04:39.339033685Z\",\n \"endTime\": \"2026-10-03T14:23:40.256240312Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-vzb878crbjw5\" to finish creating: GET https://rp.j0614912.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeadlineExceeded\n--------------------------------------------------------------------------------\n{\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"name\": \"737b799d-a4dd-42d1-a4a2-9cd870de0e20\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T14:04:39.339033685Z\",\n \"endTime\": \"2026-10-03T14:23:40.256240312Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-vzb878crbjw5--managed/providers/Microsoft.Compute/virtualMachines/e2x8w5d3p9g3r3d-np-update-nodes-mkg2d-mzpcs to exist\\n\"\n }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n \u003c[]error | len:1, cap:1\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106365799160614912","occurred_at":"2026-10-03T12:48:35Z","signature_id":"b0563ce9088e2d908cb5cdc30392d78444947ed998364dd964a7182a39daa5f6","pr_number":7340,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3528a4bdeac0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vljzr5rw54g9\" to finish creating: GET https://rp.j7922048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"name\": \"336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T20:19:49.133571808Z\",\n \"endTime\": \"2026-10-03T20:38:58.66181942Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 3 of 3 machines are not healthy\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 3 of 3 machines are not ready\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3528a4bdeac0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-vljzr5rw54g9\" to finish creating: GET https://rp.j7922048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"name\": \"336a4045-fdd2-4a2c-b138-8d99b939169e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T20:19:49.133571808Z\",\n \"endTime\": \"2026-10-03T20:38:58.66181942Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 3 of 3 machines are not healthy\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 3 of 3 machines are not ready\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-5nr55: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-8xk7g: Creating: virtualmachine creating or updating\\nMachine h9f6c5t3m0f9b9x-autoscale-az-bg5l4-kdmwk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106458234737922048","occurred_at":"2026-10-03T18:55:53Z","signature_id":"79b90ae5312ada738ac4440b4501fdd97f482ae1f84f6796798dab482e44cbd8","pr_number":7333,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"955f2e0940a6e134701e99a0c3ef85d6b5374445ed9f53e16789c85a8fd4d2bc","failure_pattern":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","search_query":"ERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/swift-vnet (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-03T12:18:04.762Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 description=\"Step swift-vnet\\n Kind: Shell\\n Command: ./swift-vnet.sh\\n\"\ntime=2026-10-03T12:18:05.175Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=swift-vnet stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: Failed to load or parse file /tmp/azure-cli-config-3770628587/azureProfile.json. It will be overridden by default settings.\\nERROR: The subscription of 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' doesn't exist in cloud 'AzureCloud'.\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7318/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106355570880548864","occurred_at":"2026-10-03T12:07:56Z","signature_id":"beba06114dcaa7d8f5cf05070efb098c564f39c87ef41d58003f0a49b2fc4e73","pr_number":7318,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-25T14:02:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x22d6ad451998\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x22d6ad451998\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47mrq.apps.aro.cilium-cluster.iobe.j9578752.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7340/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106402405829578752","occurred_at":"2026-10-03T15:14:03Z","signature_id":"dca811aa20ada3f91e9119ee0eecf021dba8048b0d56a9111868dce8ce4a50a4","pr_number":7340,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-24T14:18:18Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-03T04:27:33Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j3883264-svc\", component=\"backend\", name=\"openshiftversioncatalog\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue openshiftversioncatalog has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7332/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106223756383883264","occurred_at":"2026-10-03T03:24:09Z","signature_id":"fee414b15571075bb38e6218ae18960a82bae3ac8dd2273bf21bea06f37d99a3","pr_number":7332,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[1,5,6,7,0,3,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":43,"prior_last_seen_at":"2026-09-27T13:08:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-03T01:40:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4407168-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-74695d59f9-hxdpn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-74695d59f9-hxdpn has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-03T01:42:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4407168-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-74695d59f9-bfnvs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-74695d59f9-bfnvs has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-03T01:42:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4407168-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-74695d59f9-wjv8w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-74695d59f9-wjv8w has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106183789494407168","occurred_at":"2026-10-03T00:45:21Z","signature_id":"5772227d0b1c3dbc1f88ff19aefa92151e392000e5bb31a2f32a22c36981ef0c","pr_number":7333,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l9xkxj2bpszs\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD","STG"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":24,"prior_last_seen_at":"2026-09-27T19:23:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xd9700701640\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l9xkxj2bpszs\" to finish creating: GET https://rp.j1091968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"name\": \"d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T17:56:20.630751478Z\",\n \"endTime\": \"2026-10-03T18:15:20.975618436Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xd9700701640\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l9xkxj2bpszs\" to finish creating: GET https://rp.j1091968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"name\": \"d2ab0eb3-9aa0-4386-9325-4cdb7f9a4b9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T17:56:20.630751478Z\",\n \"endTime\": \"2026-10-03T18:15:20.975618436Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s1f2f0h0t5y3r2c-np-deletable-64c66-chds7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-l9xkxj2bpszs--managed/providers/Microsoft.Compute/virtualMachines/s1f2f0h0t5y3r2c-np-deletable-64c66-chds7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106422169931091968","occurred_at":"2026-10-03T16:32:35Z","signature_id":"a7a5e0e36a34359f8084193c2d0faab4839b075dfe3e59cc885f6fb7a3743b3d","pr_number":7333,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a8bb4542e08df91dbf01364e592eda2b949d63ea36dbf43a83cbdbd4c51b7dfe","failure_pattern":"failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","occurrences":1}],"full_error_samples":["time=2026-10-03T01:35:28.208Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-03T01:35:28.778Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy values=\"map[azureArmHelperIdentityCertName:armHelperPoolCert-10 azureArmHelperIdentityClientId:0a9e3940-6bc3-4cbf-a8f7-c65c3d8d2462 azureArmHelperMockFpaPrincipalId:47f69502-0065-4d9a-b19b-d403e183d2f4 azureCloudEnvironmentName:AzurePublicCloud azureCsMiClientId:de4f7be2-47e3-4002-adc2-77c2fcdec830 azureFirstPartyApplicationClientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX azureMiMockServicePrincipalCertName:msiMockPoolCert-17 azureMiMockServicePrincipalClientId:e7112f7f-5b45-480e-bc44-abf9f219e6f0 azureMiMockServicePrincipalPrincipalId:39eefc4f-5e24-491d-9f09-bcc0ad573bcf azureOperatorsMI:map[dev:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] public:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] roleSetName:dev] backplaneURL: batchProcesses: batchProcessesDryRun:true clientId:foo clientScopes:openid clientSecret:bar clusterErrorReport:false cosmosDatabaseName:arohcpci01-rp-j4407168 cosmosDatabaseUrl:https://arohcpci01-rp-j4407168.documents.azure.com:443/ cpuRequest:630m csDeploymentStrategy:map[rollingUpdate:map[maxSurge:1 maxUnavailable:0] type:RollingUpdate] dataPlaneHAProxyImage:arohcpocpdev.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678 databaseAuthMethod:postgres databaseDisableTls:true databaseHost:ocm-cs-db databaseK8sSecretName:ocm-cs-db databaseName:ocm-cs-db databasePassword:TheBlurstOfTimes databasePort:5432 databaseServiceName:ocm-cs-db databaseUser:ocm databaseVolumeCapacity:512Mi debugMaxDuration:5m debugPort: defaultExpiration:0 deployment:map[zoneCount:3] environment:arohcpci01 forceMigration: fpaCertName:firstPartyCert2 gatewayUrl:http://127.0.0.1:9090 global:map[imageDigest:sha256:5fffce295f8b49be38ae2385714d09af29700dd3dc60c40a058f7aeb66e97d72] imageDigest:sha256:5fffce295f8b49be38ae2385714d09af29700dd3dc60c40a058f7aeb66e97d72 imageRegistry:arohcpsvcdev.azurecr.io imageRepository:app-sre/aro-hcp-clusters-service insecure:false jwksUrl:http://localhost logLevel:4 machinePoolMigrationWorkerPeriod:1h managedIdentitiesDataPlaneAudienceResource:https://dummy.org maximumExpiration:0 memoryLimit:unlimited memoryRequest:150Mi ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev ocpAcrResourceUrl:arohcpocpdev.azurecr.io oidcIssuerBaseUrl:https://arohcpci01oidcj4407168.z9.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj4407168.blob.core.windows.net/ postgresqlVersion:12 provisionShardClusterLimit:100 pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:app-sre/aro-hcp-clusters-service:pull workloadIdentityClientId:27f533be-4c23-4032-9d19-da9c7454bd3b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] region:canadacentral replicas:3 runtimeMode:aro-hcp serviceAccountName:clusters-service serviceKeyVaultName:aro-hcp-dev-svc-kv tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesEnabled:true tlsCertificatesIssuer:Self tokenUrl:http://localhost useAzureDB:false userDefinedDnsBaseDomain:i1.devshift.org]\"\ntime=2026-10-03T01:35:28.779Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-03T01:35:29.089Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy namespace=clusters-service\ntime=2026-10-03T01:35:29.415Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy dryRun=false\ntime=2026-10-03T01:35:29.416Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-03T01:35:29.465Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-10-03T01:45:35.236Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\"\ntime=2026-10-03T01:45:35.237Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: InProgress, message: Available: 0/3\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7333/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106183789494407168","occurred_at":"2026-10-03T00:45:21Z","signature_id":"f292884abfee87bf70d297d035f626393e7b29fe680eeec4ec5586d3b8b28c52","pr_number":7333,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":6,"failed_runs":4,"raw_occurrences":37,"matched_occurrences":37,"runs_affected":4},"rows":[{"environment":"prod","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":15,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":15,"after_last_push_count":21,"trend_counts":[0,0,0,0,0,6,15],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-h26qrg\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n \u003c*exported.ResponseError | 0x34bdd7d932c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-5fxfbtp77wdj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-h26qrg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-h26qrg\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n \u003c*exported.ResponseError | 0x34bdd7d932c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-5fxfbtp77wdj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-h26qrg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x210c33ec90b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8fsbbrnx57fx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x210c33ec90b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8fsbbrnx57fx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-54csls\" to 4.21.0-0.nightly-multi-2026-10-02-115050\nUnexpected error:\n \u003c*exported.ResponseError | 0x2086109860c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-hx8x6fq8plbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-54csls\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-54csls\" to 4.21.0-0.nightly-multi-2026-10-02-115050\nUnexpected error:\n \u003c*exported.ResponseError | 0x2086109860c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-hx8x6fq8plbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-54csls\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"467ff9b3fff0fba450d4251bf1cedce098cd2ea5e773aae9c7dab251c9546c13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"b1e65d263ab4ed8d6d367a1d2b0e22cf65bd386883e59e308c33eef10b3fdd3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"d23e0af5f66a9d21ebfe7f67bab5ec265f1980b1c4649c6850acb948a72ccc0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"df520e23539da9e8ef8e4034c818cfb36af7d49a348b14cd808580a60fa3775f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"eb95f17360545876d289dee37a778611d4344608288ffa82220daa2648023378","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"0d1682a2cfbb5f2e590b6b31ccc2bdd35b4fb6efe346454c0a0465b4a4c377dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"1a76286ada45eadee61eadc06e02438d66934a94c715501b79fa55ea647992ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"366a1991051b1a37601646ae4edff31e67686f7599119c046ea638bd894e910e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"7346747ba389cbf83d28f49662dfd4c31ae1261a21f1a9894630a81cb52c0065","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"7b4cf1cf1f356f8a0bd6bfb20a926fe794e222e24e9098fbe0eb9156b1ec6573","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"8c999722171344de70dda1803d5579aed1e0dbc4ecd2b2c5ea3d0e6b792cbf9c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"919aee9941c508698346026ce9e0051090805691e68054cfc4acc83d6c2fb7d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"c17fe792fb98b2d6a545cd9ddbd5a4c5a6ed15360c664292bd1e61a9ec602478","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"d46d14f8b1e583f77549aa7ebed41d05be17337007389372864730bed2620d7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"f4f3787f7980560a701583c4b17b2aa434c0cb2c95496fa5661ef575b0ba94d2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":11,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":11,"after_last_push_count":13,"trend_counts":[0,0,0,0,0,2,11],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x325c79dc0060\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-987nc9dd4vhq\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-987nc9dd4vhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0x325c79dc0060\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-987nc9dd4vhq\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-987nc9dd4vhq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cb7bb3ff960\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pf8r4pxpb5hl\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-pf8r4pxpb5hl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cb7bb3ff960\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-pf8r4pxpb5hl\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-pf8r4pxpb5hl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ed03c0ba1c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-nrr89q\" resourcegroup=\"rg-np-dg-minor-nrr89q-d5btp9lwtkwk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-nrr89q-d5btp9lwtkwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-nrr89q/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0x1ed03c0ba1c0\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-nrr89q\" resourcegroup=\"rg-np-dg-minor-nrr89q-d5btp9lwtkwk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-nrr89q-d5btp9lwtkwk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-nrr89q/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"4cf3eda2914d3b1a9083e46adec66b3f55112482155fd5d64fc7f418ec60e0b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"ab51303944522693c0b33104b7c92d86be3eb8c3466ad20546a9b77e031ae5b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"28086d43cfa72152275388034cc851903b47aa1ca5775b0f6fa80ca38a6ab4fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"44dfa2bda4098c665bbfb261786dda1e03ddea5270c9178a60475577c209a9b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"567f7755bb3e007fee5b222ee1274b9e42bd25b0c9f732e516cf41596ecc8fab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"5bb1d3c7e546810e558daf11bf222544e4133eca40ba362ffb48b7b1423ad104","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"6db71a6e85dcae0f4cf2fff89193035c7c77e8c6a0d75a696207a46ff9701f52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"9af200308adea58cb657b883d457e0c1c8c4b68df178d893c6ee77bf70f5c283","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"a6f43bb89c4cb869e470fa6b8f415931a9ffca73b7a099b8d5d5fdb88da93883","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"c3e02910fd51fc69091b5e63232a999211acc1001c4336ec9c326557d10e1d3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"c929d59edb7567fa76ef70a100f78549e067b5211d3fa8ccbe6bb931bf8ae3bc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":3,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,5,3],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-79cvw2\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x23c456aecc90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-8lpbkjqgxtmb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-79cvw2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-79cvw2\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0x23c456aecc90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-8lpbkjqgxtmb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-79cvw2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-95bqxj\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1aa712d69710\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b5g72gdsstzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-95bqxj\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-95bqxj\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0x1aa712d69710\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-b5g72gdsstzw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-95bqxj\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-x4v9ss\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x5a0752a7c50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-9j7fg4k8csln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-x4v9ss\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-x4v9ss\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0x5a0752a7c50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-9j7fg4k8csln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-x4v9ss\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"0cf8bd41b605f3c023454134078e0b29fb24116c4ab84dc44eaed46533bbc5d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"3906d99cd657415581fb6ba0344b0053b97db85bb68b94222e88264810fdd66b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2106202761312866304","occurred_at":"2026-10-03T02:00:44Z","signature_id":"c6f98fc883c08bd20a81c2a2a9718e59825e4839ac864d806b4647a01ab008bd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":72,"also_in":["STG"],"trend_counts":[0,5,5,0,5,2,2],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":16,"prior_last_seen_at":"2026-09-26T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xcf8828ab170\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-crv2w79vmlxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xcf8828ab170\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-crv2w79vmlxc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xd97df9bad80\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cskpwn77dkwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xd97df9bad80\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-cskpwn77dkwd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"103fa735d194341e5747569e803885dd3d43db0551359672451398c565ccf594","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"46e00fd4ba016237d54a5c172dba5d7ae1620c42bcb5d60c65e763e9df2f79cd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":60,"also_in":["STG"],"trend_counts":[4,4,4,4,2,2,2],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":17,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x781abe80140\u003e: \n failed to create NodePool np-4kgdcw: failed starting nodepool creation \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\" in resourcegroup=\"rg-nightly-4-20-4t8l44cr8nbr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-4t8l44cr8nbr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-4kgdcw/nodePools/np-4kgdcw\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x781abe80140\u003e: \n failed to create NodePool np-4kgdcw: failed starting nodepool creation \"np-4kgdcw\" for cluster \"cluster-nightly-4-20-4kgdcw\" in resourcegroup=\"rg-nightly-4-20-4t8l44cr8nbr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-4t8l44cr8nbr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-4kgdcw/nodePools/np-4kgdcw\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x31d5efc20c00\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-b4wh48\" in resourcegroup=\"rg-np-version-upgrade-b4wh48-bzk7j574q45w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b4wh48-bzk7j574q45w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-b4wh48/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x31d5efc20c00\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-b4wh48\" in resourcegroup=\"rg-np-version-upgrade-b4wh48-bzk7j574q45w\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b4wh48-bzk7j574q45w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-b4wh48/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"6cf60cc34982ca354ed254e7db99f8c3c17c9ef039ff68ffc95b5e18a6f44b25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"8af9e2c89645a4bf3765a33eab4e694ab3d6205ee2dba4b2403cbbe04e4bb04b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":6,"also_in":["STG"],"trend_counts":[0,4,0,0,1,0,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33e8cc763a20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wrvrktrj9pn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"name\": \"a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:41:16.318835994Z\",\n \"endTime\": \"2026-10-03T10:00:22.618172935Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33e8cc763a20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wrvrktrj9pn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"name\": \"a021811c-f25e-4fdd-b88c-cc64a68c8188\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:41:16.318835994Z\",\n \"endTime\": \"2026-10-03T10:00:22.618172935Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k3u6n5v9b5f7j2v-np-1-9khfh-nvt5s: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106310690099695616","occurred_at":"2026-10-03T09:09:36Z","signature_id":"13af940fe84aa9225f10173afd7fedb188865ae08fb26ab20d44731489099168","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":28,"also_in":["DEV","STG"],"trend_counts":[0,4,10,2,6,5,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":20,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2861f94db110\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-s8tjmf2mg47r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"name\": \"a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:45:52.709942431Z\",\n \"endTime\": \"2026-10-03T10:04:55.390706618Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-mf948-nwnch: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-mf948-nwnch: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x2861f94db110\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-s8tjmf2mg47r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"name\": \"a70c1239-194d-44ad-b66c-18f11327f9a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T09:45:52.709942431Z\",\n \"endTime\": \"2026-10-03T10:04:55.390706618Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-mf948-nwnch: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-mf948-nwnch: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-s8tjmf2mg47r--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-mf948-nwnch to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106310690099695616","occurred_at":"2026-10-03T09:09:36Z","signature_id":"3bc3377f053adf5e02635272297b8adacea15f1260ea4f1c4bf49a21872672c8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n \u003c*errors.joinError | 0x29a4ab6ac030\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:241]: failed to verify HCP cluster viability\nUnexpected error:\n \u003c*errors.joinError | 0x29a4ab6ac030\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106519855229308928","occurred_at":"2026-10-03T23:00:45Z","signature_id":"8128113059a95965bef3d3c445b7c0cac5fe7442cb1f7985e9426b72d3507e7d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jdf825hs8fn9\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV","STG"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":11,"prior_last_seen_at":"2026-09-25T23:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3a380b9ff820\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jdf825hs8fn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"name\": \"1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T10:22:50.222769152Z\",\n \"endTime\": \"2026-10-03T10:41:50.376552801Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x3a380b9ff820\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jdf825hs8fn9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"name\": \"1ffbe111-707b-4771-8ddd-d413913d8bde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T10:22:50.222769152Z\",\n \"endTime\": \"2026-10-03T10:41:50.376552801Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jdf825hs8fn9--managed/providers/Microsoft.Compute/virtualMachines/f0m1e1q3s5q9h4n-np-deletable-cb4h4-4ccd6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106302957979963392","occurred_at":"2026-10-03T08:38:53Z","signature_id":"adee8926a72629f9d64b7a77f2665eb3670a4af4a352e102d836fb3757b3bc93","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":4,"failed_runs":4,"raw_occurrences":38,"matched_occurrences":37,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":23,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":23,"after_last_push_count":78,"also_in":["PROD"],"trend_counts":[0,0,0,0,8,47,23],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34ef387fcea0\u003e: \n failed to create NodePool np-vgspf2: failed waiting for nodepool=\"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\" in resourcegroup=\"rg-nightly-4-21-dsfs76js9854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"name\": \"78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:34:31.068400483Z\",\n \"endTime\": \"2026-10-03T23:53:40.358231747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: Creating: virtualmachine creating or updating\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34ef387fcea0\u003e: \n failed to create NodePool np-vgspf2: failed waiting for nodepool=\"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\" in resourcegroup=\"rg-nightly-4-21-dsfs76js9854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"name\": \"78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:34:31.068400483Z\",\n \"endTime\": \"2026-10-03T23:53:40.358231747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: Creating: virtualmachine creating or updating\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x31a06c9a5d00\u003e: \n failed to create NodePool np-dz68ws: failed waiting for nodepool=\"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\" in resourcegroup=\"rg-nightly-5-0-4t5m8n4czrvj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"name\": \"2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:35:55.566084896Z\",\n \"endTime\": \"2026-10-03T23:55:00.222357543Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: Creating: virtualmachine creating or updating\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x31a06c9a5d00\u003e: \n failed to create NodePool np-dz68ws: failed waiting for nodepool=\"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\" in resourcegroup=\"rg-nightly-5-0-4t5m8n4czrvj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"name\": \"2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:35:55.566084896Z\",\n \"endTime\": \"2026-10-03T23:55:00.222357543Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: Creating: virtualmachine creating or updating\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x366855210620\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4c8kl482c2st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\",\n \"name\": \"37548aa4-a4e7-458d-b686-fb1201996386\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:15:28.253854505Z\",\n \"endTime\": \"2026-10-03T23:34:30.095088976Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: Creating: virtualmachine creating or updating\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x366855210620\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4c8kl482c2st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\",\n \"name\": \"37548aa4-a4e7-458d-b686-fb1201996386\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:15:28.253854505Z\",\n \"endTime\": \"2026-10-03T23:34:30.095088976Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: Creating: virtualmachine creating or updating\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"390cd3c454160d58c3f66648e63a7907c2041ab6f7cec42614cfb83642a9075a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"5c797d19c80475dc8cd016a2f12b5969e5f66674a64c327fd5b8176cbb909c6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"88bfecb223eed4d7950743d5a839fc1088854870e36ffeb70cfcf0e0e832d4ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"d2485400b9fb14ab84f4a84285253177801b00cc9d4149c5818d0c29ddef9be2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"e9efc6e08915eb27cce0e9b2aee48e6761e029a06f9f351f49af927ae38ed282","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"f9761523403367d45961176e4cd187bb59f7a5f7a10bfd1b755e1c02f0042a38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"2c2caa46b0c9a860c8536cb1d0cfe4dc435fe9c87395f2009a2b50d36e765530","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"2c9f39e0ab026a6227defad21130e9e9d394ff6a055a7d1fa925b7fc0262141f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"6ed585f3f050ad8b54d850a6ae0fdd9b4909a19d9c493dbc159c6d4911096dd3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"8ed18e43fabdf08315daac2d8e02c94f2c8a172b70d38853c874dbc955f5d982","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"9528ad897be3c780f7f61e03b1ee8a6fe0b00f376a6ff34c98cd8ee75346769c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"e5c35e2b9b861c774ee1789f184fbf1cb2f5f308d8dc05c180346b85e646fe05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"e9fa9ae8e858a7b5355cb1b43d0d6779abb4d6d970a6d15f0440c7b03e009e78","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"fd808e83579b395ae5abd54c9de0aa37f2d5bc4ae6a53ba2e695ba0fa5c2dbc8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"06c296f07902755728beb3838eb1ec4ac170f38778b421b87c5fc204e64d1fe4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"173a7e9edaf6f61e13589f056617df6ddb63fa543ce96dbe851cba6372717a02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"2270ab4d2942cd093aff353fd34037a69c93e335ec7f1a88dfa214a3f4004534","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"2a69c285def7ab9febd458730c74909c9a16be32934cbdbd5ca44844262b1eb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"34a4dddd4ac406094ca0f1a4bf7eb674580835c68e05a723aea1b94befa47471","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"38fdd980bf52332969e2e4f977878b2b3caffd7b67413af3dedd10df203bfc1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"399671fb721e085227d1bcef5acf1cf7744be28e89d36f981b42cc7c1ec8a4d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"8f5972671bc46aef70b454f5d37eb1f821ffe511b2716373992fdcfdd6e793dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"ed4c47b0094dced5127989ca5fda7f6a4352c3e90ee689cf14e5def8764d9f1d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":5,"after_last_push_count":37,"also_in":["PROD"],"trend_counts":[0,5,5,0,0,2,5],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-26T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x36252fbe4930\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-548xlmq4vp84/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x36252fbe4930\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-548xlmq4vp84/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x336e50b8a360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-njsbz992h4xx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x336e50b8a360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-njsbz992h4xx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3102b57c8300\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-5xj4fb5fc6dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3102b57c8300\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-5xj4fb5fc6dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"0c43de85fc2d6a7c071c55d1f96008747d85103a5e49f90377c2b68757383d44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"1c86c022cf7d0c7808be0cd45d82052e35c88c4c35fb78fb4a2b9cf8b10f84d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"4220fe7603791b9ddfb09840fb8cd4a59017f25f10351d3998d45dda9b1f0d93","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"9a1223f045ad2c43a2d2b9f5c9d15450c0250574d5337dd1a988d4371b014517","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"e723e28bcd7e435dcc3421287fec024c49aa82e18f5dc42262402d55af5c17bd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":3,"after_last_push_count":3,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":12,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x104897805d20\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r8db5d4ldfmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"name\": \"12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:41:40.611562032Z\",\n \"endTime\": \"2026-10-04T00:00:46.799966407Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: Creating: virtualmachine creating or updating\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x104897805d20\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r8db5d4ldfmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"name\": \"12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:41:40.611562032Z\",\n \"endTime\": \"2026-10-04T00:00:46.799966407Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: Creating: virtualmachine creating or updating\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x933c5f12a60\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lbkmsp45kpxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\",\n \"name\": \"efe85861-3de4-4231-94f4-c87043439307\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:54:52.260057343Z\",\n \"endTime\": \"2026-10-03T04:13:53.608229594Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: Creating: virtualmachine creating or updating\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x933c5f12a60\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lbkmsp45kpxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\",\n \"name\": \"efe85861-3de4-4231-94f4-c87043439307\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:54:52.260057343Z\",\n \"endTime\": \"2026-10-03T04:13:53.608229594Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: Creating: virtualmachine creating or updating\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xca0cd836f80\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"name\": \"4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:51:49.926221366Z\",\n \"endTime\": \"2026-10-03T02:10:56.26372719Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: Creating: virtualmachine creating or updating\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xca0cd836f80\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"name\": \"4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:51:49.926221366Z\",\n \"endTime\": \"2026-10-03T02:10:56.26372719Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: Creating: virtualmachine creating or updating\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"2b24f7ee1c45683523e52ab6a6fd28bf4368385c2a3884bc30ecbb5e5516b086","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"882fba6cad22093956999ee2384f3228d45b1777a330537ec945b37008289a0a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"d6f5de42eefb7152643e950681559e711013d084e28baf1d882e6f029c0e5511","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":44,"also_in":["DEV","PROD"],"trend_counts":[1,2,1,10,6,5,2],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":23,"prior_last_seen_at":"2026-09-27T02:00:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x15f1e13c9ee0\u003e: \n failed to create NodePool np-6tqvsp: failed waiting for nodepool=\"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\" in resourcegroup=\"rg-candidate-4-21-qxp54ksqdkvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"name\": \"cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:23:37.870437581Z\",\n \"endTime\": \"2026-10-03T03:42:43.397028044Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: Creating: virtualmachine creating or updating\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x15f1e13c9ee0\u003e: \n failed to create NodePool np-6tqvsp: failed waiting for nodepool=\"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\" in resourcegroup=\"rg-candidate-4-21-qxp54ksqdkvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"name\": \"cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:23:37.870437581Z\",\n \"endTime\": \"2026-10-03T03:42:43.397028044Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: Creating: virtualmachine creating or updating\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xbbf668ab320\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lqwbkg\" in resourcegroup=\"rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"name\": \"a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:58:23.770970582Z\",\n \"endTime\": \"2026-10-03T02:17:26.319658092Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xbbf668ab320\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lqwbkg\" in resourcegroup=\"rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"name\": \"a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:58:23.770970582Z\",\n \"endTime\": \"2026-10-03T02:17:26.319658092Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"8dc98ddc487d792e30d36e88cd34b10e8b904d76ef343c1f573a64f712f3b220","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"d5c37afd72eeef3b1717f0ed626a3a3f20e451ad1fbcbdcf4ca6127c60afb433","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":2,"after_last_push_count":30,"also_in":["PROD"],"trend_counts":[1,4,4,4,0,2,2],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x242124721260\u003e: \n failed to create NodePool np-zb9djq: failed starting nodepool creation \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\" in resourcegroup=\"rg-nightly-4-20-r55xk7zmjr9c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-r55xk7zmjr9c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zb9djq/nodePools/np-zb9djq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x242124721260\u003e: \n failed to create NodePool np-zb9djq: failed starting nodepool creation \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\" in resourcegroup=\"rg-nightly-4-20-r55xk7zmjr9c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-r55xk7zmjr9c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zb9djq/nodePools/np-zb9djq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x184038744400\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tzvzfh\" in resourcegroup=\"rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tzvzfh/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x184038744400\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tzvzfh\" in resourcegroup=\"rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tzvzfh/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"4e5c1277a0285514fbe13ffd0e118b84890287d84121f73af4e99275508596ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"80a54d00d33387405efc62cd65294145e6838630c336a95f7908f0dc39f21afd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":7,"also_in":["DEV"],"trend_counts":[0,0,0,0,1,5,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1363e14cc018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-jtmvgv9m6829\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"name\": \"31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:01:05.764058109Z\",\n \"endTime\": \"2026-10-03T01:20:12.852653102Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1363e14cc018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-jtmvgv9m6829\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"name\": \"31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:01:05.764058109Z\",\n \"endTime\": \"2026-10-03T01:20:12.852653102Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"475946b6879259ac2394beb241c800f706ea9c7b18c2b17cfb61377bbdc2cf11","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xe7412ec40a8\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-j2962r94fd6x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"name\": \"4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:00:19.581988952Z\",\n \"endTime\": \"2026-10-03T03:19:19.770602537Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xe7412ec40a8\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-j2962r94fd6x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"name\": \"4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:00:19.581988952Z\",\n \"endTime\": \"2026-10-03T03:19:19.770602537Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"39751860e6d26d74a5f620ec93a78ea99d0e96ecd0ed7587dea64af7ba46a985","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}