{"meta":{"start_date":"2026-10-08","end_date":"2026-10-08","start_at":"2026-10-08T00:00:00Z","end_at":"2026-10-09T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T12:09:39Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":91,"failed_runs":66,"raw_occurrences":117,"matched_occurrences":111,"runs_affected":60},"rows":[{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":19,"runs_affected":19,"failed_runs":19,"run_impact_percent":20.87912087912088,"anchor_occurrences":19,"after_last_push_count":68,"trend_counts":[19,15,3,26,15,29,19],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":191,"prior_last_seen_at":"2026-10-04T23:41:00Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":19}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T23:11:31Z\n Ended: 2026-10-08T23:17:33Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0504704-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T23:16:25Z\n Ended: 2026-10-08T23:32:25Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3544576-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T19:18:13Z\n Ended: 2026-10-08T19:24:08Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8343424-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317197410504704","occurred_at":"2026-10-08T22:02:45Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7389,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7180,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256251258343424","occurred_at":"2026-10-08T18:00:34Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256213476052992","occurred_at":"2026-10-08T18:00:25Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7389,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7389,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7401/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108163022403932160","occurred_at":"2026-10-08T11:50:06Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":7401,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108153534082977792","occurred_at":"2026-10-08T11:12:24Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7383,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152878592954368","occurred_at":"2026-10-08T11:09:46Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":7364,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152449343688704","occurred_at":"2026-10-08T11:08:06Z","signature_id":"795e6b05034ec7d4a1f17a125759873597c9a2e2ed36b762a41eae274e90b610","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108130674367008768","occurred_at":"2026-10-08T09:41:34Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7288,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108128762271895552","occurred_at":"2026-10-08T09:33:58Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7387,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7420/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108123255498870784","occurred_at":"2026-10-08T09:12:04Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7420,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108110245078241280","occurred_at":"2026-10-08T08:20:23Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108043334072143872","occurred_at":"2026-10-08T03:54:28Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7177,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108031501605015552","occurred_at":"2026-10-08T03:07:29Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7412,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108018145892503552","occurred_at":"2026-10-08T02:14:25Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108015628659986432","occurred_at":"2026-10-08T02:04:25Z","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","pr_number":7232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":12,"runs_affected":11,"failed_runs":11,"run_impact_percent":12.087912087912088,"anchor_occurrences":12,"after_last_push_count":195,"trend_counts":[21,10,61,276,236,126,12],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":198,"prior_last_seen_at":"2026-10-04T23:52:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 5.0 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2b5cwp\" for cluster \"cluster-candidate-5-1-2b5cwp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x36065ae1e700\u003e: \n failed to create NodePool np-2b5cwp: failed waiting for nodepool=\"np-2b5cwp\" for cluster \"cluster-candidate-5-1-2b5cwp\" in resourcegroup=\"rg-candidate-5-1-mdbffxtcx8wr\" to finish creating: GET https://rp.j6247936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\",\n \"name\": \"e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T19:25:31.299024517Z\",\n \"endTime\": \"2026-10-08T19:54:32.257674658Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-mdbffxtcx8wr-candidate-2b5cwp--managed/providers/Microsoft.Compute/virtualMachines/e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-mdbffxtcx8wr-candidate-2b5cwp--managed/providers/Microsoft.Compute/virtualMachines/e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2b5cwp\" for cluster \"cluster-candidate-5-1-2b5cwp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x36065ae1e700\u003e: \n failed to create NodePool np-2b5cwp: failed waiting for nodepool=\"np-2b5cwp\" for cluster \"cluster-candidate-5-1-2b5cwp\" in resourcegroup=\"rg-candidate-5-1-mdbffxtcx8wr\" to finish creating: GET https://rp.j6247936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\",\n \"name\": \"e3f9fab1-4935-49c2-8e86-66ac4db0e7cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T19:25:31.299024517Z\",\n \"endTime\": \"2026-10-08T19:54:32.257674658Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-mdbffxtcx8wr-candidate-2b5cwp--managed/providers/Microsoft.Compute/virtualMachines/e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-mdbffxtcx8wr-candidate-2b5cwp--managed/providers/Microsoft.Compute/virtualMachines/e5w2m4n7q0t5w6r-np-2b5cwp-krd4h-vff46 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d9f3c92fc0\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-gcnbkpsxtjrv\" to finish creating: GET https://rp.j5951872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/511caec2-c197-4b9c-9a43-b8c00295c7d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/511caec2-c197-4b9c-9a43-b8c00295c7d1\",\n \"name\": \"511caec2-c197-4b9c-9a43-b8c00295c7d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T20:56:10.667754986Z\",\n \"endTime\": \"2026-10-08T21:25:14.560841698Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6 to exist\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6 to exist\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:117]: failed to create node pool cordon-np\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d9f3c92fc0\u003e: \n failed to create NodePool cordon-np: failed waiting for nodepool=\"cordon-np\" for cluster \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-gcnbkpsxtjrv\" to finish creating: GET https://rp.j5951872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/511caec2-c197-4b9c-9a43-b8c00295c7d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/511caec2-c197-4b9c-9a43-b8c00295c7d1\",\n \"name\": \"511caec2-c197-4b9c-9a43-b8c00295c7d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T20:56:10.667754986Z\",\n \"endTime\": \"2026-10-08T21:25:14.560841698Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6 to exist\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-gn6m6 to exist\\nMachine e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-gcnbkpsxtjrv--managed/providers/Microsoft.Compute/virtualMachines/e1k5o2y0i4g4i1e-cordon-np-dpcqc-kfh5v to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-n4rtkb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e49d4d5a340\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-n4rtkb\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj\" to finish creating: GET https://rp.j0335104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0463f50-61f7-4515-9352-022d3d794646\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0463f50-61f7-4515-9352-022d3d794646\",\n \"name\": \"b0463f50-61f7-4515-9352-022d3d794646\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:41:51.052187772Z\",\n \"endTime\": \"2026-10-08T17:10:57.967451326Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj-cp-ystream-den-6158ac1e/providers/Microsoft.Compute/virtualMachines/r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj-cp-ystream-den-6158ac1e/providers/Microsoft.Compute/virtualMachines/r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-n4rtkb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e49d4d5a340\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-n4rtkb\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj\" to finish creating: GET https://rp.j0335104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0463f50-61f7-4515-9352-022d3d794646\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0463f50-61f7-4515-9352-022d3d794646\",\n \"name\": \"b0463f50-61f7-4515-9352-022d3d794646\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:41:51.052187772Z\",\n \"endTime\": \"2026-10-08T17:10:57.967451326Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj-cp-ystream-den-6158ac1e/providers/Microsoft.Compute/virtualMachines/r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-zvs4xjzt6ggj-cp-ystream-den-6158ac1e/providers/Microsoft.Compute/virtualMachines/r4e3r4w5k9d2k3l-np-deny-pods-z474z-5l4nk to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108259306926247936","occurred_at":"2026-10-08T18:12:42Z","signature_id":"9bbb9a304b3e216bd163fb308a5bc70d9beeba6ca91eeefd3ac9063123943dc7","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256216495951872","occurred_at":"2026-10-08T18:00:26Z","signature_id":"e52340d4b382cdd2542ada160fb67461b260ba9f52c641a312b0e9e87e132a0b","pr_number":7387,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7431/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108220296300335104","occurred_at":"2026-10-08T15:37:42Z","signature_id":"d5485ecbaf29eb6153d7283048b59ef21ec89644b21cb87a56aa54d40e6a3fa5","pr_number":7431,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108209233152446464","occurred_at":"2026-10-08T14:53:44Z","signature_id":"9f5e90ecbae9c619ce85e90de1ac44b213ae43628780d197bc24085fef9e391f","pr_number":7361,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","signature_id":"6089f291ebd2776303b1bbe50c4d8aa3d633aed985bf844efdbe7aacda4c2397","pr_number":7389,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152449343688704","occurred_at":"2026-10-08T11:08:06Z","signature_id":"1cc5d051835a1c8eb905dcafe82dc37af6a69b3130687b92e4a9ee1058c03a32","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108129603208876032","occurred_at":"2026-10-08T09:37:19Z","signature_id":"b1cc99febe4b197af72ea8a773b460d06bc2296c0c3a577338c7f8d560c7e86f","pr_number":7158,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108122723405271040","occurred_at":"2026-10-08T09:09:57Z","signature_id":"167a399ee5497032fbc211b91c5977c4d716b98bb1d5acb6868aa44e16e76145","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108122723405271040","occurred_at":"2026-10-08T09:09:57Z","signature_id":"8a3ed87cf84ff42b79a1fd8e8f951dc027047229d3c301b92c6142a018727cb3","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108109354774302720","occurred_at":"2026-10-08T08:16:51Z","signature_id":"f22d3abbd1cab4d50eb4fc487acaabfea9c0b84dab46bd31dda1987b2400c11a","pr_number":7389,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108092712069632000","occurred_at":"2026-10-08T07:10:43Z","signature_id":"5fc986fabe0026f4b77c8908e8c0e313a1116526b8618c4e0b755089134b6b09","pr_number":7412,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108043334072143872","occurred_at":"2026-10-08T03:54:28Z","signature_id":"f21ba692b551caebe9fb47bce79bd94b5286f66a20a64edd5fe17a6b67fb55c5","pr_number":7177,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":8,"runs_affected":8,"failed_runs":8,"run_impact_percent":8.791208791208792,"anchor_occurrences":8,"after_last_push_count":12,"trend_counts":[2,0,0,6,5,4,8],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":34,"prior_last_seen_at":"2026-10-02T20:20:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","occurrences":8}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T00:16:50Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0504704-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0504704/providers/microsoft.monitor/accounts/services-j0504704\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-5fdbbd5c5b-l6bt7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j0504704-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T00:16:50Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0504704-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0504704/providers/microsoft.monitor/accounts/services-j0504704\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-5fdbbd5c5b-l6bt7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j0504704-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T00:17:23Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4990592-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.37:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4990592/providers/microsoft.monitor/accounts/services-j4990592\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-c59bdf7c8-spr9r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j4990592-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T00:17:23Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4990592-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.37:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4990592/providers/microsoft.monitor/accounts/services-j4990592\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-c59bdf7c8-spr9r\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j4990592-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T20:22:48Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j8343424-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.84:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8343424/providers/microsoft.monitor/accounts/services-j8343424\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-68b89b8957-rg9qs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j8343424-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T20:23:48Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j8343424-svc\", cluster_type=\"svc-cluster\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.84:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8343424/providers/microsoft.monitor/accounts/services-j8343424\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-68b89b8957-rg9qs\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j8343424-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317197410504704","occurred_at":"2026-10-08T22:02:45Z","signature_id":"478c229f000373303094e502fe5867ab11bb46818c55fe4914e070c3ce22b112","pr_number":7389,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317114874990592","occurred_at":"2026-10-08T22:02:25Z","signature_id":"23b4c1fcca9c095c3310a93e598a980361364ffd7335fed71ecdd85117fddf7e","pr_number":7364,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256251258343424","occurred_at":"2026-10-08T18:00:34Z","signature_id":"4f9f8db4cb3f25b70a751ee41026225f752fc0d2a21726d68e44cf77af9d8f9f","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108248385097043968","occurred_at":"2026-10-08T17:29:18Z","signature_id":"293fb0dcb3ed0945c0a68b097fa102ed57356450154c2fa182d4761ee486de09","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7162/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108221120938250240","occurred_at":"2026-10-08T15:40:58Z","signature_id":"27e681f75202e09c64bf1bc1b442c563ec7f3137ae918747ec39eefd467cec74","pr_number":7162,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108181128597737472","occurred_at":"2026-10-08T13:01:55Z","signature_id":"71513c99645f5a7b46d22a8d1780149946d3f4d432f0cff96d7e41203c90b083","pr_number":7387,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152449343688704","occurred_at":"2026-10-08T11:08:06Z","signature_id":"b3859c09791b6a1dee073ca99210a9b01905c300d49723024d404dda969c5d31","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108130674367008768","occurred_at":"2026-10-08T09:41:34Z","signature_id":"0505f57a41fb3f2b5cc475246473e4503d1def678146b025f446120e420b23ae","pr_number":7288,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":7.6923076923076925,"anchor_occurrences":7,"after_last_push_count":24,"trend_counts":[1,0,0,7,7,2,7],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":69,"prior_last_seen_at":"2026-10-02T10:59:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":7}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T00:04:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3153024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.30:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3153024/providers/microsoft.monitor/accounts/hcps-j3153024\", namespace=\"ocm-arohcpci01-2tat52el7ut34bj4g2ggf0vutik7dd90-j3v2u2r2o5q0x4m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-tbmch\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat52el7ut34bj4g2ggf0vutik7dd90-j3v2u2r2o5q0x4m/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T00:04:29Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3153024-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.30:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3153024/providers/microsoft.monitor/accounts/hcps-j3153024\", namespace=\"ocm-arohcpci01-2tat52el7ut34bj4g2ggf0vutik7dd90-j3v2u2r2o5q0x4m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-tbmch\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat52el7ut34bj4g2ggf0vutik7dd90-j3v2u2r2o5q0x4m/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 6 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T00:00:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.220:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vt6m4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T00:00:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.220:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vt6m4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-09T00:00:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.220:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vt6m4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-09T00:00:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.220:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-vt6m4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat5kosr5birnmo003enu4ag9tvkblp-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-09T00:16:57Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-sst2s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-09T00:17:58Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-sst2s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T19:58:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6247936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.232:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6247936/providers/microsoft.monitor/accounts/hcps-j6247936\", namespace=\"ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-6l8pk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T19:59:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6247936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.232:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6247936/providers/microsoft.monitor/accounts/hcps-j6247936\", namespace=\"ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-6l8pk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-08T20:07:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6247936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"csi-snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.232:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6247936/providers/microsoft.monitor/accounts/hcps-j6247936\", namespace=\"ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-6l8pk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q/csi-snapshot-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-08T20:07:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6247936-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"csi-snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.232:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6247936/providers/microsoft.monitor/accounts/hcps-j6247936\", namespace=\"ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-6l8pk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tapqhcmll32asjjo23us2mqtb5k3kur-b2t0m2c3a0t5y9q/csi-snapshot-controller has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317188833153024","occurred_at":"2026-10-08T22:02:43Z","signature_id":"c904114929467bb32c67ef275024818849dff8d5a227072eff3a386882ff848e","pr_number":7292,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","signature_id":"4bef60ee24c495084afb20bb4a919adeb209d9fae5a39e972a9d5a8c119a1d43","pr_number":7180,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108259306926247936","occurred_at":"2026-10-08T18:12:42Z","signature_id":"7e6534085d42dc6118c7a37dc3076a5592621ad332aa0f0dc2d28d01dd263625","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256213476052992","occurred_at":"2026-10-08T18:00:25Z","signature_id":"481b22a2ae9014aee1f61475b4be429b12de22c8d096e16cf3cf96337a687bf6","pr_number":7389,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108197422965460992","occurred_at":"2026-10-08T14:06:48Z","signature_id":"b50a31093336d8331ae783c4d9e7666f297d09f74d347cd9133e1c15e42f6f1b","pr_number":7422,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108153534082977792","occurred_at":"2026-10-08T11:12:24Z","signature_id":"2bb5f552ea27ec9022c5035d204ea8f422b813234e438a587a8af153658a6644","pr_number":7383,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"7e1b30717a7056324794286c92396616dc0d26090203314168033730260fac49","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aec9b94fcdd3a2f33bc17cf1885b9f3b0b0ea550986aec929b8e75820929123a","failure_pattern":"cluster \u003cresource-group\u003e/\u003ccluster\u003e entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","search_query":"cluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":7.6923076923076925,"anchor_occurrences":7,"after_last_push_count":13,"trend_counts":[0,0,1,4,6,5,7],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":23,"prior_last_seen_at":"2026-10-04T22:58:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":7}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-7fcmrmvxxv7n/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-7fcmrmvxxv7n/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-ttr5gf9j2z4p/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-ttr5gf9j2z4p/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster delayed-rbac-4tll47xvzcmb/delayed-rbac-cluster entered terminal state \"Failed\" after role assignment deployment; expected \"Succeeded\"\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: Failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317119643914240","occurred_at":"2026-10-08T22:02:26Z","signature_id":"eaa2cead8693174d0c57211952caffdb2c67d10b7dc850a37b0ecc67cb24793b","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108291662634553344","occurred_at":"2026-10-08T20:21:17Z","signature_id":"108516cb0647ba3d2f4c87e8257ac1527a6e7121e308f5f5579382713182c90a","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256210653286400","occurred_at":"2026-10-08T18:00:24Z","signature_id":"75c025839b4b77bebd3fbfe7d5be33a670bec55dc16b0fc470c5b7fcd3c0ad20","pr_number":7180,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108207385897078784","occurred_at":"2026-10-08T14:46:24Z","signature_id":"7a3a06bb18ed74825abe049cd09dc707ee7a940daddfb1a541b6df012fba70c9","pr_number":7105,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108109354774302720","occurred_at":"2026-10-08T08:16:51Z","signature_id":"ffdb06d45831bbbf8246e86634c1c7fa86a1828ef1864f846896e0d1bc90fa06","pr_number":7389,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108104508235059200","occurred_at":"2026-10-08T07:57:36Z","signature_id":"b122c2e9da305b502bb36b2c546534409e8af338dfddda201b8e60b513ad641d","pr_number":6775,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"b132c43fd39d685c72ce0a5f22659a72c1f81a5d0a9dd5f857638ee06b041704","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1bdb9911aad76e13f69b1ac4294cfb812df237ced9319f2fd48ec7be614a885c","failure_pattern":"alert [svc] LeaderElectionLeaseStale fired","search_query":"alert [svc] LeaderElectionLeaseStale fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":6.593406593406593,"anchor_occurrences":6,"after_last_push_count":8,"trend_counts":[0,0,0,5,6,3,6],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":22,"prior_last_seen_at":"2026-09-18T15:17:47Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert LeaderElectionLeaseStale does not fire","occurrences":6}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T20:07:32Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j5978496-mgmt-1\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.233:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5978496/providers/microsoft.monitor/accounts/services-j5978496\", namespace=\"klusterlet-2tapn8dlbj9p40f8ecef335ah1n44qgl\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-snrs6\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tapn8dlbj9p40f8ecef335ah1n44qgl on cluster ci01-j5978496-mgmt-1 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T18:47:49Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9147264-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.232:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9147264/providers/microsoft.monitor/accounts/services-j9147264\", namespace=\"klusterlet-2taok2013r1ut7fvo0ulo5kc534ntq7s\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-fljb2\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2taok2013r1ut7fvo0ulo5kc534ntq7s on cluster ci01-j9147264-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T13:48:14Z\n Severity: Sev3\n Labels: alertname=\"LeaderElectionLeaseStale\", cluster=\"ci01-j9653888-mgmt-2\", component=\"leader-election\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.155:8080\", job=\"kube-state-metrics\", lease=\"work-manager\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9653888/providers/microsoft.monitor/accounts/services-j9653888\", namespace=\"klusterlet-2tak8n7sjj3mgsij1id8amijusdtem6a\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-87ql4\", prometheus=\"prometheus/prometheus\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Leader election lease work-manager in namespace klusterlet-2tak8n7sjj3mgsij1id8amijusdtem6a on cluster ci01-j9653888-mgmt-2 has not been renewed for more than 37 minutes. The leadership election might be broken or the component stopped running."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256282895978496","occurred_at":"2026-10-08T18:00:39Z","signature_id":"5a7e6f416bdd814aa6842042551270c54b63893bd3ddd65b8fe96afa706bcb83","pr_number":7361,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","signature_id":"9a5683b6fb4e970df594707347166c3a5464fde6e4de17ed87d6680996438cdc","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","signature_id":"13fd8fddd2d9f41c17b15476dbb1a6d18d3c125423d8872d5b4d0619fa486992","pr_number":7389,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108152878592954368","occurred_at":"2026-10-08T11:09:46Z","signature_id":"e82e51d2e215239d7b4cd935e809cb5992c44a5d389b3abdcf1a7fc37def35f8","pr_number":7364,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7378/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108101483491758080","occurred_at":"2026-10-08T07:45:34Z","signature_id":"afb48a8584f79bbdd2eee32217731be00e00c63d7874b7427fe12a549c0c7d4a","pr_number":7378,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108092712069632000","occurred_at":"2026-10-08T07:10:43Z","signature_id":"17502e6a782e32cf8d05639323afa25f0eb69d1c67087752a4cd2f50e385cc87","pr_number":7412,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.2967032967032965,"anchor_occurrences":4,"after_last_push_count":109,"trend_counts":[0,0,0,0,0,3,4],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":155,"prior_last_seen_at":"2026-09-30T18:13:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1044cd538270\u003e: \n resource group \"rg-hs-pre--l7mzsgckn5bq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-m4xdbw\" in resourcegroup=\"rg-hs-pre--l7mzsgckn5bq\" to finish deleting: GET https://rp.j8864256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f1c7e5-bd06-48b6-8e00-126748584d33\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f1c7e5-bd06-48b6-8e00-126748584d33\",\n \"name\": \"81f1c7e5-bd06-48b6-8e00-126748584d33\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T22:52:33.320294122Z\",\n \"endTime\": \"2026-10-08T23:16:35.834245337Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tasnkqnpkt06eq3lmg9dsk6e9purnb2 still exists (deletion dispatched at 2026-10-08T22:52:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1044cd538270\u003e: \n resource group \"rg-hs-pre--l7mzsgckn5bq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-m4xdbw\" in resourcegroup=\"rg-hs-pre--l7mzsgckn5bq\" to finish deleting: GET https://rp.j8864256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f1c7e5-bd06-48b6-8e00-126748584d33\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/81f1c7e5-bd06-48b6-8e00-126748584d33\",\n \"name\": \"81f1c7e5-bd06-48b6-8e00-126748584d33\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T22:52:33.320294122Z\",\n \"endTime\": \"2026-10-08T23:16:35.834245337Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tasnkqnpkt06eq3lmg9dsk6e9purnb2 still exists (deletion dispatched at 2026-10-08T22:52:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3adaf74760a8\u003e: \n resource group \"fips-enabled-khqp8vkktpzt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-khqp8vkktpzt\" to finish deleting: GET https://rp.j2446464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/407d6cdf-c346-4727-b3ca-9d3137b70cc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/407d6cdf-c346-4727-b3ca-9d3137b70cc4\",\n \"name\": \"407d6cdf-c346-4727-b3ca-9d3137b70cc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:13:27.229044406Z\",\n \"endTime\": \"2026-10-08T16:37:32.626253468Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tamus8q8drvpor1j4v73qnjn2pbcpag still exists (deletion dispatched at 2026-10-08T16:13:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x3adaf74760a8\u003e: \n resource group \"fips-enabled-khqp8vkktpzt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-khqp8vkktpzt\" to finish deleting: GET https://rp.j2446464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/407d6cdf-c346-4727-b3ca-9d3137b70cc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/407d6cdf-c346-4727-b3ca-9d3137b70cc4\",\n \"name\": \"407d6cdf-c346-4727-b3ca-9d3137b70cc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:13:27.229044406Z\",\n \"endTime\": \"2026-10-08T16:37:32.626253468Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tamus8q8drvpor1j4v73qnjn2pbcpag still exists (deletion dispatched at 2026-10-08T16:13:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1aa3a56380a8\u003e: \n resource group \"private-kas-mmbkx8tpnrlv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-mmbkx8tpnrlv\" to finish deleting: GET https://rp.j2446464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bb91148d-7554-4a55-b1dd-4854fa37b494\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bb91148d-7554-4a55-b1dd-4854fa37b494\",\n \"name\": \"bb91148d-7554-4a55-b1dd-4854fa37b494\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:22:19.235799645Z\",\n \"endTime\": \"2026-10-08T16:46:22.547532555Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tamuoo5fqaudkk7q5op01b86l5utmqh still exists (deletion dispatched at 2026-10-08T16:22:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1aa3a56380a8\u003e: \n resource group \"private-kas-mmbkx8tpnrlv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-mmbkx8tpnrlv\" to finish deleting: GET https://rp.j2446464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bb91148d-7554-4a55-b1dd-4854fa37b494\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bb91148d-7554-4a55-b1dd-4854fa37b494\",\n \"name\": \"bb91148d-7554-4a55-b1dd-4854fa37b494\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:22:19.235799645Z\",\n \"endTime\": \"2026-10-08T16:46:22.547532555Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tamuoo5fqaudkk7q5op01b86l5utmqh still exists (deletion dispatched at 2026-10-08T16:22:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108311064448864256","occurred_at":"2026-10-08T21:38:22Z","signature_id":"87480de3ded9bd25bdf3480fe553bfa947aa5a80228ef448c08c54751c457c7d","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108209233152446464","occurred_at":"2026-10-08T14:53:44Z","signature_id":"0e4b25ccea79ae9ce774b1501e6333b6a44367fc8f0e789e5ce189b4d807484c","pr_number":7361,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108209233152446464","occurred_at":"2026-10-08T14:53:44Z","signature_id":"7cbece22670a9f172523e3b354735cd562db62f7e42f6e03ee9317ebdd207c57","pr_number":7361,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108110245078241280","occurred_at":"2026-10-08T08:20:23Z","signature_id":"d6f3dbb1bf15144cc0a62926bd3e48a5cae1b7b619360a81ed11fa37f85d80e3","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.197802197802198,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-29T14:09:17Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","occurrences":2},{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert collection is complete","occurrences":2}],"full_error_samples":["alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7950976/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j7950976' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108246398544646144","occurred_at":"2026-10-08T17:21:25Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7422,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108246398544646144","occurred_at":"2026-10-08T17:21:25Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7422,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108169795277950976","occurred_at":"2026-10-08T12:16:52Z","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","pr_number":7226,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108169795277950976","occurred_at":"2026-10-08T12:16:52Z","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","pr_number":7226,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.2967032967032965,"anchor_occurrences":3,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,4,3],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-29T20:58:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xed922aab58\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0xed9195f1e0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0xed91fd22c0\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xed922aab58\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0xed9195f1e0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0xed91fd22c0\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0x7b70a422fc0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0x7b70a4e87a0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0x7b70a6fbd20\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0x7b70a422fc0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0x7b70a4e87a0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0x7b70a6fbd20\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0x1e29e0f51bc0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0x1e29e2b846a0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0x1e29e176e4f0\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0x1e29e0f51bc0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n {\n errs: [\n \u003c*fmt.wrapError | 0x1e29e2b846a0\u003e{\n msg: \"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\",\n err: \u003c*errors.errorString | 0x1e29e176e4f0\u003e{\n s: \"IngressController loadBalancer config is nil\",\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108228889548427264","occurred_at":"2026-10-08T16:11:50Z","signature_id":"a43f99b00f1bbd4a520785c52599fad0e47aab9b8859a5d6eb181cda1291d1d6","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108090608831696896","occurred_at":"2026-10-08T07:02:22Z","signature_id":"3cc6e00f4b908acb0ea62e41044f8a94e6ce5698be960fa38a7f85470bd35cba","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108018145892503552","occurred_at":"2026-10-08T02:14:25Z","signature_id":"38cafc0c9d53e48809e4f80070514d02cab460de26c498c2ee86e36255eb9c93","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.2967032967032965,"anchor_occurrences":3,"after_last_push_count":68,"trend_counts":[5,2,0,3,6,9,3],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":155,"prior_last_seen_at":"2026-10-03T16:57:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T21:07:52Z\n Ended: 2026-10-08T21:13:54Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5951872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tar6kfvsbfkgd4sfo2s5ak9up9mghnd-ephemeral-disk\", pod=\"openshift-apiserver-f7bf46fd6-jsjg9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tar6kfvsbfkgd4sfo2s5ak9up9mghnd-ephemeral-disk/openshift-apiserver-f7bf46fd6-jsjg9 has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T18:18:15Z\n Ended: 2026-10-08T18:25:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9147264-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2taok4lfsu899t43cjah6nqtn60rbr6d-r8d4l6w3w4r4h6t\", pod=\"openshift-apiserver-59b5875fdc-r9dd5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2taok4lfsu899t43cjah6nqtn60rbr6d-r8d4l6w3w4r4h6t/openshift-apiserver-59b5875fdc-r9dd5 has been in a non-ready state for longer than 5 minutes.","alert fired 57 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-85f67c86d5-gvdlf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-85f67c86d5-gvdlf has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-pj78n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-pj78n has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-jnjtd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-jnjtd has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-rplxb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-rplxb has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:36:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-htmxm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-htmxm has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-rv6cc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-rv6cc has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"cloud-network-config-controller-74769fccfb-k8kwp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/cloud-network-config-controller-74769fccfb-k8kwp has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-hgjs7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-hgjs7 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-85f67c86d5-gs5jc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-85f67c86d5-gs5jc has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-8chb5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-8chb5 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:36:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-psbgs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-psbgs has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-4tbx4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-4tbx4 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-gjpp8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-gjpp8 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-j44kk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-j44kk has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:36:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-mvbj5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-mvbj5 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-ss6rg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-ss6rg has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-snf5p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-snf5p has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-mdgsc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-mdgsc has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-85f67c86d5-cwkvx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-85f67c86d5-cwkvx has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-bnmrb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-bnmrb has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-cv4g6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-cv4g6 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-7n652\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-7n652 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-4hlqb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-4hlqb has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-w2spp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-w2spp has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-87cjv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-87cjv has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-ht9c9\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-ht9c9 has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:36:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-jwnrm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-jwnrm has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-8mmlh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-8mmlh has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-7pzw8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-7pzw8 has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-672v2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-672v2 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-9c8996664-xqlcz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-9c8996664-xqlcz has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-f8gbx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-f8gbx has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-l29xs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-l29xs has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-mcs4j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-mcs4j has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-m8sbd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-m8sbd has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-fg868\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-fg868 has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-d57l7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-d57l7 has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-6qjzf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-6qjzf has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-ssj25\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-ssj25 has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:36:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-2nvzx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-2nvzx has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-4rxvd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-4rxvd has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-l6dz6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-l6dz6 has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-6qjld\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-6qjld has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-85f67c86d5-m9rtt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-85f67c86d5-m9rtt has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-2njx7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-2njx7 has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-2t8lh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-2t8lh has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-10-08T04:06:15Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-rhccp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-rhccp has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-g64ns\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-g64ns has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-5c9wj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-5c9wj has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-kdtjv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-kdtjv has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-5fld5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-5fld5 has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-skdmq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-skdmq has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-lfqpn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-lfqpn has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:16Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-85f67c86d5-kq58q\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-85f67c86d5-kq58q has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-pjnm6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-pjnm6 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-file-csi-driver-controller-c955dbc8f-hztch\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-file-csi-driver-controller-c955dbc8f-hztch has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-10-08T04:06:16Z\n Ended: 2026-10-08T04:31:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5015552-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a\", pod=\"azure-disk-csi-driver-controller-d4df64db4-zgk6p\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tac93jjbc7p84jirpisg4dmju2f1ram-w3k7u8o1f6o9c5a/azure-disk-csi-driver-controller-d4df64db4-zgk6p has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108256216495951872","occurred_at":"2026-10-08T18:00:26Z","signature_id":"1e60a482d86ac260f86e568bf243b409c0a25d681cfd32fd6d4ce13302592f5a","pr_number":7387,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","signature_id":"dea7649e1e363ea4afbe251966c45c45a904d878aa58e50d8b09b9fb6f157106","pr_number":7232,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108031501605015552","occurred_at":"2026-10-08T03:07:29Z","signature_id":"33a563d1f9854fc5c2d805dce229ec37f3683cf9f20862a546be44d8addf4f52","pr_number":7412,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.197802197802198,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,1,2,1,2],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c27e024540\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-6zjcv2t4zcfg\" to finish creating: GET https://rp.j0298368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\",\n \"name\": \"7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:08:53.200082246Z\",\n \"endTime\": \"2026-10-08T14:38:00.180447244Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1c27e024540\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-6zjcv2t4zcfg\" to finish creating: GET https://rp.j0298368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\",\n \"name\": \"7fdab8b1-d3ef-4d1c-89fb-9da45a5eaf01\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T14:08:53.200082246Z\",\n \"endTime\": \"2026-10-08T14:38:00.180447244Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:444]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc4b0a06e918\u003e: \n failed to create NodePool agg-np-b: failed waiting for nodepool=\"agg-np-b\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-qnvb5555k8tb\" to finish creating: GET https://rp.j8011648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\",\n \"name\": \"7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:02:19.757714703Z\",\n \"endTime\": \"2026-10-08T04:31:24.163355482Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:444]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc4b0a06e918\u003e: \n failed to create NodePool agg-np-b: failed waiting for nodepool=\"agg-np-b\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-qnvb5555k8tb\" to finish creating: GET https://rp.j8011648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\",\n \"name\": \"7b6e27a8-980f-45cb-ad53-cd8329dd5c4a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T04:02:19.757714703Z\",\n \"endTime\": \"2026-10-08T04:31:24.163355482Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7403/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108182743610298368","occurred_at":"2026-10-08T13:08:28Z","signature_id":"2ab2afb758add7ee8595134f90992b6ec46e194592da1dee4e4a8da09cd82ffb","pr_number":7403,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7361/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025089978011648","occurred_at":"2026-10-08T02:42:01Z","signature_id":"898639e46d512657722f4281a7d0dfb4428d8ed1240fc047c4055963519f6d5a","pr_number":7361,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b821a370667d0a6279446842808eaf289925d243f648908c207a98a9eafaa561","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ErrCode_InsufficientVCPUQuota; detail message Insufficient vcpu quota requested \u003ccount\u003e, remaining \u003ccount\u003e for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: \u003curl\u003e Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.197802197802198,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-21T16:54:04Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":2}],"full_error_samples":["time=2026-10-08T16:19:05.365Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-10-08T16:30:17.342Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T16:19:05.375275155Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T16:19:08.114395254Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T16:19:08.462375095Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335389178Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"6F2FEEE9F3FAC221\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:19:14.8133695Z\\\",\\\"duration\\\":\\\"PT9M5.9123598S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335532727Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"98034B352C012EE7\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT4.7536001S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.33558093Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DE129F3B0C485FAF\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT1M38.1712414S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335599002Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"B325BEA3E7A07E7A\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT1M56.0129258S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335620623Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"D7A0CEADFD719B73\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:28:15.8471301Z\\\",\\\"duration\\\":\\\"PT3.9567105S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335657835Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"7F61B179B809D260\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:19:14.8133695Z\\\",\\\"duration\\\":\\\"PT9M0.9591654S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:30:17.335709239Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5048320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j5048320-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584101305363180932\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft2\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft1\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5048320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j5048320-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 72 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"11m11.960448525s\\\"}\\n exit status 1\"","time=2026-10-08T16:14:31.581Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-10-08T16:22:39.946Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T16:14:31.590606721Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T16:14:33.50353792Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T16:14:33.891298669Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944584337Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"241855BF9A0D266F\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT1M38.1681956S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944653829Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"5B81E75ADA721336\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT4.9210349S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.94468287Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"0A9A6CB955E8A5E9\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:14:38.0622167Z\\\",\\\"duration\\\":\\\"PT7M50.9531763S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944693871Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"127F3C3755EDC782\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT4.2491544S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944712361Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1AF780C01A639DF0\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-10-08T16:20:32.6231206Z\\\",\\\"duration\\\":\\\"PT4.2335315S\\\",\\\"statusCode\\\":\\\"BadRequest\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"ErrCode_InsufficientVCPUQuota\\\",\\\"message\\\":\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944730542Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2DBAB0983C574AF5\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-10-08T16:14:38.0622167Z\\\",\\\"duration\\\":\\\"PT5M54.4662985S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-10-08T16:22:39.944752483Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3418880-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3418880-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584101308111130568\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft1\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"BadRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3418880-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3418880-mgmt-1/agentPools/userswft2\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"{\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"code\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"ErrCode_InsufficientVCPUQuota\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"details\\\\\\\\\\\\\\\": null,\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"message\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"Insufficient vcpu quota requested 80, remaining 0 for family StandardDdsv6Family for region westus3. If you want to increase the quota, please follow this instruction: https://learn.microsoft.com/en-us/azure/quotas/view-quotas. Surge nodes would also consume vcpu quota, please consider use smaller maxSurge or use maxUnavailable to proceed upgrade without surge nodes, details: aka.ms/aks/maxUnavailable.\\\\\\\\\\\\\\\",\\\\\\\\r\\\\\\\\n \\\\\\\\\\\\\\\"subcode\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\"\\\\\\\\r\\\\\\\\n}\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"8m8.354161621s\\\"}\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108224410405048320","occurred_at":"2026-10-08T15:54:02Z","signature_id":"664aa8f1aeb2650939331a96787558c2e7340b55e45ef2847dd30061d3213023","pr_number":6928,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108222712823418880","occurred_at":"2026-10-08T15:47:18Z","signature_id":"94dd4a2e46776ab0e7c5e9b444cb5e7c4246df9918134350ba21a74752e8098b","pr_number":7364,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.197802197802198,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,2,1,0,2],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-18T15:17:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2c4b80612378\u003e: \n resource group \"ingress-cert-le-68vr9hbvh2st\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-68vr9hbvh2st\" to finish deleting: GET https://rp.j3544576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca06aaf0-429b-4724-9342-8885fc527050\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca06aaf0-429b-4724-9342-8885fc527050\",\n \"name\": \"ca06aaf0-429b-4724-9342-8885fc527050\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T23:18:31.808957419Z\",\n \"endTime\": \"2026-10-08T23:42:33.737902461Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tat5kosr5birnmo003enu4ag9tvkblp still exists (deletion dispatched at 2026-10-08T23:18:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2c4b80612378\u003e: \n resource group \"ingress-cert-le-68vr9hbvh2st\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-68vr9hbvh2st\" to finish deleting: GET https://rp.j3544576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca06aaf0-429b-4724-9342-8885fc527050\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca06aaf0-429b-4724-9342-8885fc527050\",\n \"name\": \"ca06aaf0-429b-4724-9342-8885fc527050\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T23:18:31.808957419Z\",\n \"endTime\": \"2026-10-08T23:42:33.737902461Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tat5kosr5birnmo003enu4ag9tvkblp still exists (deletion dispatched at 2026-10-08T23:18:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x126c34d780d8\u003e: \n resource group \"rg-np-version-upgrade-d2kz66-n2vxth4prxxw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-d2kz66\" in resourcegroup=\"rg-np-version-upgrade-d2kz66-n2vxth4prxxw\" to finish deleting: GET https://rp.j8427264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\",\n \"name\": \"53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:58:56.699498743Z\",\n \"endTime\": \"2026-10-08T18:23:02.391989177Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tao6fb6en2d65csg2b2sr0ei58fvhru still exists (deletion dispatched at 2026-10-08T17:58:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x126c34d780d8\u003e: \n resource group \"rg-np-version-upgrade-d2kz66-n2vxth4prxxw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-d2kz66\" in resourcegroup=\"rg-np-version-upgrade-d2kz66-n2vxth4prxxw\" to finish deleting: GET https://rp.j8427264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\",\n \"name\": \"53f3ea8e-927e-4f49-ab09-36a6f2f31c6f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:58:56.699498743Z\",\n \"endTime\": \"2026-10-08T18:23:02.391989177Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tao6fb6en2d65csg2b2sr0ei58fvhru still exists (deletion dispatched at 2026-10-08T17:58:57Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","signature_id":"b9572568edad6e488a7ae0195c7620f5eca412060505066f9072aea06ab756cc","pr_number":7180,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108228889548427264","occurred_at":"2026-10-08T16:11:50Z","signature_id":"8db99276b1e37ca2b73c952dfac288a0b7ae6c24b3ed28e03f5e617ef6f4ab9c","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c42f860bbf4cb76b671dc080497cd48bd19935942783e87b46a282ef4f701461","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"engineering-workers\": (must be a valid DNS RFC 1035 label); provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.197802197802198,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,2,2],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x14550d98c9a0\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-p98fbdr6lfb6\": PUT https://rp.j6887424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p98fbdr6lfb6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x14550d98c9a0\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-p98fbdr6lfb6\": PUT https://rp.j6887424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-p98fbdr6lfb6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x1657ab71ce40\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-rkxd84qqqgg8\": PUT https://rp.j9463680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rkxd84qqqgg8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:102]: failed to create workers for SWIFT outbound controls\nUnexpected error:\n \u003c*fmt.wrapError | 0x1657ab71ce40\u003e: \n failed to create NodePool engineering-workers: failed starting nodepool creation \"engineering-workers\" for cluster \"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-rkxd84qqqgg8\": PUT https://rp.j9463680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-rkxd84qqqgg8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster/nodePools/engineering-workers\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: MultipleErrorsOccurred\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"MultipleErrorsOccurred\",\n \"message\": \"2 errors occurred\",\n \"details\": [\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Too long: may not be more than 15 bytes\",\n \"target\": \"id\"\n },\n {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"engineering-workers\\\": (must be a valid DNS RFC 1035 label)\",\n \"target\": \"id\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108316538086887424","occurred_at":"2026-10-08T22:00:07Z","signature_id":"7e465cdb4b8db885d2f1d5a2350e3f71d22bf5070b017c7982cfbb0f329e79aa","pr_number":7397,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7397/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108178486299463680","occurred_at":"2026-10-08T12:51:33Z","signature_id":"703ff58f90b61266d47daa64e91c49cac059b2fddfc664d613b54ef26eabfb02","pr_number":7397,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert collection is complete","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.197802197802198,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-29T14:09:17Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [infra] alert collection is complete","occurrences":2}],"full_error_samples":["alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-usw3/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-usw3' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7950976: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7950976/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j7950976' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108246398544646144","occurred_at":"2026-10-08T17:21:25Z","signature_id":"a13b64d94d488082a4a9ffd8fdeb35b5d5d0a71c93381c3fedbe0bba79b488d1","pr_number":7422,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108169795277950976","occurred_at":"2026-10-08T12:16:52Z","signature_id":"2859658063bed829d162181cb3d76e39201e6bf4d59aca2d6c51092fd846a233","pr_number":7226,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"fa106de26c29607201ec4aa06c503cd9db99f65136c9944baef8962062de09a9","failure_pattern":"VerifyHyperShiftIngressCertificate timed out after 10m0","search_query":"VerifyHyperShiftIngressCertificate timed out after 10m0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.197802197802198,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,2],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should serve the default ingress certificate and allow a customer certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for 5.1","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xef3ce514120\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xef3ce514100\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xef3ce5140e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xef3ce514120\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.007s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xef3ce514100\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0xef3ce5140e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1d63e63ce540\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d63e63ce520\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d63e63ce4e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:435]: HyperShift's ingress certificate secret must exist and be referenced by IngressController/default\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1d63e63ce540\u003e: \n VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\n {\n msg: \"VerifyHyperShiftIngressCertificate timed out after 10m0.001s: getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d63e63ce520\u003e{\n msg: \"getting HyperShift ingress secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1d63e63ce4e0\u003e{\n msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108090608831696896","occurred_at":"2026-10-08T07:02:22Z","signature_id":"06adbd5f174da4a722aae0d056ec2b5a7c54b016aa9e689d5cee91e57df2195d","pr_number":7325,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108018145892503552","occurred_at":"2026-10-08T02:14:25Z","signature_id":"dc991bbf770e80e3e153a619922727dc6f068b8152a0521def1711d4a491b244","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.197802197802198,"anchor_occurrences":2,"after_last_push_count":16,"trend_counts":[1,0,0,2,5,1,2],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":37,"prior_last_seen_at":"2026-10-02T10:59:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":2}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-09T00:01:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"cloud-network-config-controller-79b774bd6b-9pkrz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6b4e41e1-2282-44a1-ab27-d2b7bffdbb7d\"\n Description: Pod ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/cloud-network-config-controller-79b774bd6b-9pkrz (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-09T00:01:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"csi-snapshot-controller-7fcf9f696b-5dd5v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ade41fab-9ae3-4c1c-af6d-d25dc46782e7\"\n Description: Pod ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/csi-snapshot-controller-7fcf9f696b-5dd5v (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-09T00:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"csi-snapshot-controller-7fcf9f696b-5dd5v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ade41fab-9ae3-4c1c-af6d-d25dc46782e7\"\n Description: Pod ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/csi-snapshot-controller-7fcf9f696b-5dd5v (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-09T00:02:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3544576-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.130:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3544576/providers/microsoft.monitor/accounts/hcps-j3544576\", namespace=\"ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster\", pod=\"cloud-network-config-controller-79b774bd6b-9pkrz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6b4e41e1-2282-44a1-ab27-d2b7bffdbb7d\"\n Description: Pod ocm-arohcpci01-2tat6eoal9j7k661kv4fcopvbul662l3-basic-cluster/cloud-network-config-controller-79b774bd6b-9pkrz (controller) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T06:00:31Z\n Ended: 2026-10-08T06:22:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4542976-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4542976/providers/microsoft.monitor/accounts/hcps-j4542976\", namespace=\"ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9e83f444-711c-486c-b496-8152e0711540\"\n Description: Pod ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-08T06:00:31Z\n Ended: 2026-10-08T06:22:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4542976-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4542976/providers/microsoft.monitor/accounts/hcps-j4542976\", namespace=\"ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9e83f444-711c-486c-b496-8152e0711540\"\n Description: Pod ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7180/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317092523544576","occurred_at":"2026-10-08T22:02:20Z","signature_id":"2d0718dd6bc944ecb1f8e90e435ead1ad5ebb1657342b129e97e84fc23cf0c63","pr_number":7180,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"e76d176cb58c095e589bc218c3e104c756d14a6ceb09fa88d761f62736c4f607","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.197802197802198,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,1,2,1,2],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-25T10:20:52Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T15:04:42Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j1446656-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T15:05:40Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j1446656-mgmt-1\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T14:57:45Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j4452224-mgmt-2\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108199393621446656","occurred_at":"2026-10-08T14:14:35Z","signature_id":"22ea0c3803bb98326df9cbf23de00a80472321a99d7d574f14fa44cb6663b005","pr_number":7414,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7427/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108191395574452224","occurred_at":"2026-10-08T13:42:51Z","signature_id":"4f5e43f8961ce1450724da4818b03afc4ea4a1e65b095d58ea6cf9018f4c082d","pr_number":7427,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[1,0,0,2,3,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":25,"prior_last_seen_at":"2026-10-02T10:59:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xa118982a360\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dx54wv2l48hl\" to finish creating: GET https://rp.j4542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54ed370f-4e3b-4fe8-92dd-a7f286d6f761\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54ed370f-4e3b-4fe8-92dd-a7f286d6f761\",\n \"name\": \"54ed370f-4e3b-4fe8-92dd-a7f286d6f761\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T05:35:12.262324182Z\",\n \"endTime\": \"2026-10-08T06:04:17.39866058Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xa118982a360\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dx54wv2l48hl\" to finish creating: GET https://rp.j4542976.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54ed370f-4e3b-4fe8-92dd-a7f286d6f761\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/54ed370f-4e3b-4fe8-92dd-a7f286d6f761\",\n \"name\": \"54ed370f-4e3b-4fe8-92dd-a7f286d6f761\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T05:35:12.262324182Z\",\n \"endTime\": \"2026-10-08T06:04:17.39866058Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"09cb0962695c90bbcb8aa045ebb144f397afbd0455023f7fea0160df7e60cafc","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d1a2d3dadcb1c305fd461d5e9526c1e1f98093bc5d6545af22d4aa0cf7e7d759","failure_pattern":"ERROR CODE: DeploymentFailed; detail code NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/velero' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-08T03:34:00.150Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-10-08T03:34:00.987Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-10-08T03:34:02.023Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=058182c57c37b78b09387fdf90aeaea50b1d526f842d6127dc4d5ebd76617220 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j9563392-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F058182c57c37b78b09387fdf90aeaea50b1d526f842d6127dc4d5ebd76617220\ntime=2026-10-08T03:36:02.640Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.Resources/deployments/058182c57c37b78b09387fdf90aeaea50b1d526f842d6127dc4d5ebd76617220/operationStatuses/08584101764439358601\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-post\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/velero' was not found.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/mgmt-agent' was not found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"NotFound\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9563392-mgmt-1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/velero' was not found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025152309563392","occurred_at":"2026-10-08T02:42:16Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,1,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-28"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-04T11:57:13Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x18aecfdf0aa0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dq2w5s6mnvnb\" to finish creating: GET https://rp.j9632000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n \"name\": \"5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T08:17:53.86568203Z\",\n \"endTime\": \"2026-10-08T08:18:02.050800903Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x18aecfdf0aa0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-dq2w5s6mnvnb\" to finish creating: GET https://rp.j9632000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n \"name\": \"5591fbb4-44d5-4678-9638-e11ff2fb9020\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T08:17:53.86568203Z\",\n \"endTime\": \"2026-10-08T08:18:02.050800903Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7412/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108092712069632000","occurred_at":"2026-10-08T07:10:43Z","signature_id":"99a86419200aac49952e3148c95bd65fe8b3fbe3e68a7b83763b8f9aa7967171","pr_number":7412,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,3,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":15,"prior_last_seen_at":"2026-09-28T08:18:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2e1efc3180c0\u003e: \n resource group \"admin-api-serialconsole-bootdiag-2vphmzvmmzbh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-2vphmzvmmzbh\" to finish deleting: GET https://rp.j4279424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\",\n \"name\": \"7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:17:07.770334262Z\",\n \"endTime\": \"2026-10-08T16:41:10.588488129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tan1stnb9qf5nfuks8bh5mvhk8g2eul still exists (deletion dispatched at 2026-10-08T16:17:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x2e1efc3180c0\u003e: \n resource group \"admin-api-serialconsole-bootdiag-2vphmzvmmzbh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-2vphmzvmmzbh\" to finish deleting: GET https://rp.j4279424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\",\n \"name\": \"7a1b3c13-6b24-46ac-a6bd-8eadaaab84aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:17:07.770334262Z\",\n \"endTime\": \"2026-10-08T16:41:10.588488129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tan1stnb9qf5nfuks8bh5mvhk8g2eul still exists (deletion dispatched at 2026-10-08T16:17:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108214686754279424","occurred_at":"2026-10-08T15:15:21Z","signature_id":"80c9f90b48cd6de845d1ea9fcfc0ec8a6a97df54de2e0676d98bab3ffdbc932e","pr_number":7305,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"82fcc2cca51beebf00c7302e8f8dc3563b7995a4550dbcf53d6d84d1820ed342","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-17T08:16:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1ae233ff2078\u003e: \n resource group \"v20260901preview-hqg6wzs5tgtz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-hqg6wzs5tgtz\" to finish deleting: GET https://rp.j2977792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\",\n \"name\": \"b7f165f5-05d4-46b6-9b64-0134316e8951\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T12:28:19.710418085Z\",\n \"endTime\": \"2026-10-08T12:52:26.28519628Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tajia53q41l8grpaq7k8eu3qvn81ojn still exists (deletion dispatched at 2026-10-08T12:28:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1ae233ff2078\u003e: \n resource group \"v20260901preview-hqg6wzs5tgtz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-hqg6wzs5tgtz\" to finish deleting: GET https://rp.j2977792.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b7f165f5-05d4-46b6-9b64-0134316e8951\",\n \"name\": \"b7f165f5-05d4-46b6-9b64-0134316e8951\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T12:28:19.710418085Z\",\n \"endTime\": \"2026-10-08T12:52:26.28519628Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tajia53q41l8grpaq7k8eu3qvn81ojn still exists (deletion dispatched at 2026-10-08T12:28:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108153534082977792","occurred_at":"2026-10-08T11:12:24Z","signature_id":"2fd4ac18224462c0996aff2443f5feed4b4cd3d6b83054edc878630f518b2ab6","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0cce093122447f5c3b6f99be8e0de298d1fbee72db77c1703737ae037ec0882b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xd2898b7c180\u003e: \n resource group \"private-kas-f55dwxz4lh8f\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-f55dwxz4lh8f\" to finish deleting: GET https://rp.j8241280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n \"name\": \"f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:06:03.074876755Z\",\n \"endTime\": \"2026-10-08T10:30:10.535297041Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tahe6f6jvv1r5au11pgmqub5ah0te26 still exists (deletion dispatched at 2026-10-08T10:06:03Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xd2898b7c180\u003e: \n resource group \"private-kas-f55dwxz4lh8f\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-f55dwxz4lh8f\" to finish deleting: GET https://rp.j8241280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n \"name\": \"f521bfd4-8e62-4bc6-af54-f7aeca38d8ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T10:06:03.074876755Z\",\n \"endTime\": \"2026-10-08T10:30:10.535297041Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tahe6f6jvv1r5au11pgmqub5ah0te26 still exists (deletion dispatched at 2026-10-08T10:06:03Z); [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108110245078241280","occurred_at":"2026-10-08T08:20:23Z","signature_id":"a341e9bf98d3c6591ee21f0e758f438f31604f2f35d4323eac348c53761f7bb0","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"1a53b0c30b515bff265c32e9b65ff52445c52775e0172c5df823b4597bd77e2b","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1c054b45c180\u003e: \n resource group \"rg-candidate-4-21-l7wm67p4w7mm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-pjnsh8\" in resourcegroup=\"rg-candidate-4-21-l7wm67p4w7mm\" to finish deleting: GET https://rp.j8427264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\",\n \"name\": \"de22b374-9b97-455f-b396-0c997a7b79ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:28:41.684717507Z\",\n \"endTime\": \"2026-10-08T17:52:48.530098677Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x1c054b45c180\u003e: \n resource group \"rg-candidate-4-21-l7wm67p4w7mm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-pjnsh8\" in resourcegroup=\"rg-candidate-4-21-l7wm67p4w7mm\" to finish deleting: GET https://rp.j8427264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de22b374-9b97-455f-b396-0c997a7b79ac\",\n \"name\": \"de22b374-9b97-455f-b396-0c997a7b79ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T17:28:41.684717507Z\",\n \"endTime\": \"2026-10-08T17:52:48.530098677Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod networks (1), namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7325/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108228889548427264","occurred_at":"2026-10-08T16:11:50Z","signature_id":"7ca525a80ef321179f5efc8d43f4c69d8ec239f2d58e9a339c8fd79d95ff3fb1","pr_number":7325,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-17T08:16:38Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x44478a7c150\u003e: \n resource group \"arm64-vm-cluster-ldnw4gs8dwrw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-ldnw4gs8dwrw\" to finish deleting: GET https://rp.j7078784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcf937e7-61c6-4778-aabd-893fd37924c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcf937e7-61c6-4778-aabd-893fd37924c1\",\n \"name\": \"fcf937e7-61c6-4778-aabd-893fd37924c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:17:57.9406496Z\",\n \"endTime\": \"2026-10-08T16:41:57.233664583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0x44478a7c150\u003e: \n resource group \"arm64-vm-cluster-ldnw4gs8dwrw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-ldnw4gs8dwrw\" to finish deleting: GET https://rp.j7078784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcf937e7-61c6-4778-aabd-893fd37924c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fcf937e7-61c6-4778-aabd-893fd37924c1\",\n \"name\": \"fcf937e7-61c6-4778-aabd-893fd37924c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T16:17:57.9406496Z\",\n \"endTime\": \"2026-10-08T16:41:57.233664583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108207385897078784","occurred_at":"2026-10-08T14:46:24Z","signature_id":"13da5141ccb9f5269ca39d7846b3675b49c796a19d5d81feddf02f4d488d863f","pr_number":7105,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[1,0,0,1,1,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":12,"prior_last_seen_at":"2026-10-02T23:57:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1dd12c4c3a80\u003e: \n failed to create NodePool np-ft6nng: failed waiting for nodepool=\"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\" in resourcegroup=\"rg-candidate-4-22-j25t2jm5j66l\" to finish creating: GET https://rp.j9390848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1dd12c4c3a80\u003e: \n failed to create NodePool np-ft6nng: failed waiting for nodepool=\"np-ft6nng\" for cluster \"cluster-candidate-4-22-ft6nng\" in resourcegroup=\"rg-candidate-4-22-j25t2jm5j66l\" to finish creating: GET https://rp.j9390848.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1bff9fd2-6231-420f-ace0-63d110f1901e\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7383/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108217820029390848","occurred_at":"2026-10-08T15:27:51Z","signature_id":"a9e7d98d59a4178bf41e7b5661530ffe552aba4c92d51d87d0a3a1f5a9752100","pr_number":7383,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7855bddb074e5ee4da59762c697f802a687000bfd9477e9f92fc724e190b9330","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-\u003cid\u003e in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-9c9j4)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-17T12:57:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n \u003c*errors.joinError | 0x3e609f8540a8\u003e: \n VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-9c9j4)\n {\n errs: [\n \u003c*fmt.wrapError | 0x3e609fd10180\u003e{\n msg: \"VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*fmt.wrapError | 0x3e609fd10160\u003e{\n msg: \"failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*fmt.wrapError | 0x3e609fd10020\u003e{\n msg: \"failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*errors.StatusError | 0x3e609eee2c80\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n Reason: \"InternalError\",\n Details: {\n Name: \"fips-check-9c9j4\",\n Group: \"\",\n Kind: \"pods\",\n UID: \"\",\n Causes: [\n {Type: ..., Message: ..., Field: ...},\n ],\n RetryAfterSeconds: 0,\n },\n Code: 500,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n \u003c*errors.joinError | 0x3e609f8540a8\u003e: \n VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-9c9j4)\n {\n errs: [\n \u003c*fmt.wrapError | 0x3e609fd10180\u003e{\n msg: \"VerifyFIPSEnabled failed: failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*fmt.wrapError | 0x3e609fd10160\u003e{\n msg: \"failed to check FIPS mode on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4: failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*fmt.wrapError | 0x3e609fd10020\u003e{\n msg: \"failed to get pod logs for pod fips-check-9c9j4 in namespace e2e-fips-test-8h4th on node i8p2v2r0i5k3k0h-np-1-w6zbx-n7md4 after retries: an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n err: \u003c*errors.StatusError | 0x3e609eee2c80\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"an error on the server (\\\"unknown\\\") has prevented the request from succeeding (get pods fips-check-9c9j4)\",\n Reason: \"InternalError\",\n Details: {\n Name: \"fips-check-9c9j4\",\n Group: \"\",\n Kind: \"pods\",\n UID: \"\",\n Causes: [\n {Type: ..., Message: ..., Field: ...},\n ],\n RetryAfterSeconds: 0,\n },\n Code: 500,\n },\n },\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7288/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108025105396273152","occurred_at":"2026-10-08T02:42:04Z","signature_id":"b6ae0ddb16ff4c94221800e11be0c27c614a44b85087bb2bbb64f1a59d3dee88","pr_number":7288,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":8,"prior_last_seen_at":"2026-10-03T15:14:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x1409e4db1080\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x1409e4324dc0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1409e4f4c060\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1409e4f4c040\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0x1409e4db1080\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n errs: [\n \u003c*fmt.wrapError | 0x1409e4324dc0\u003e{\n msg: \"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1409e4f4c060\u003e{\n msg: \"DNS for route host agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1409e4f4c040\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-76bcc.apps.aro.cilium-cluster.brv0.j5059200.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108104508235059200","occurred_at":"2026-10-08T07:57:36Z","signature_id":"99e7ae876a6bc4905e749326067bbde5a6d33b2d8c46d8ed6fbac42e335e17f3","pr_number":6775,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":22,"trend_counts":[1,0,0,2,2,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":49,"prior_last_seen_at":"2026-10-02T10:59:29Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T05:54:34Z\n Ended: 2026-10-08T06:18:34Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4542976-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4542976/providers/microsoft.monitor/accounts/hcps-j4542976\", namespace=\"ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-g29zr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-08T05:54:34Z\n Ended: 2026-10-08T06:19:33Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4542976-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.202:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4542976/providers/microsoft.monitor/accounts/hcps-j4542976\", namespace=\"ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-g29zr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tadr4bm6brdefckebujuar6b3a54pbe-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108055963964542976","occurred_at":"2026-10-08T04:44:42Z","signature_id":"85682e6aa72ffd8e2527c0a0e4f27430fd467da3643a75af116ad2157337fe40","pr_number":7414,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":21,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":39,"prior_last_seen_at":"2026-10-02T23:00:08Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-08T23:31:34Z\n Ended: 2026-10-09T00:04:31Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j4990592-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j4990592-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108317114874990592","occurred_at":"2026-10-08T22:02:25Z","signature_id":"9dd9aa3c94a6d8b3b046f06122de3e26069f01c693ef0bd9d5a924fe0d0972a0","pr_number":7364,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[3,1,0,4,5,2,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":55,"prior_last_seen_at":"2026-10-03T00:45:21Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 11 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-2g4tl\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-2g4tl has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-l5mn9\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-l5mn9 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-vfshb\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-vfshb has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-qhgdx\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-qhgdx has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-wm888\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-wm888 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-t9gmx\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-t9gmx has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-h7lkm\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-h7lkm has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-w5s5h\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-w5s5h has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-vwbm7\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-vwbm7 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-n644b\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-n644b has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-08T19:06:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8981120-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5ddf5fbfd4-gh9b9\", region=\"westus3\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5ddf5fbfd4-gh9b9 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7433/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108258997998981120","occurred_at":"2026-10-08T18:11:29Z","signature_id":"da0348e89e81da96d29b328c4681e84b359b3ea85a39dbf681fbeba48167e0dd","pr_number":7433,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":8,"trend_counts":[0,0,0,2,1,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":15,"prior_last_seen_at":"2026-09-29T16:52:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3dc0df3e260\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x3dc0df3e260\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:52922-\u003e172.30.0.10:53: read: connection refused\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7389/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108165661279653888","occurred_at":"2026-10-08T12:00:36Z","signature_id":"65b7540a6ca0b846f6cc9d88ae9a4e6476ca721e437f95cf3292bc42c3aa02d4","pr_number":7389,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"12b919ef45af23cca2352cbc0e1bd15984998d1d53ba9c82192d5ec6cbeed5c5","failure_pattern":"console should return a successful response or redirect, got 503","search_query":"console should return a successful response or redirect, got 503","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":4,"prior_last_seen_at":"2026-10-02T11:44:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:312]: Timed out after 2700.006s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:310 with:\nconsole should return a successful response or redirect, got 503\nExpected\n \u003cint\u003e: 503\nto be \u003c\n \u003cint\u003e: 400\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:312]: Timed out after 2700.006s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:310 with:\nconsole should return a successful response or redirect, got 503\nExpected\n \u003cint\u003e: 503\nto be \u003c\n \u003cint\u003e: 400"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108189567159898112","occurred_at":"2026-10-08T13:35:35Z","signature_id":"7f9cdf151f9d6c520b9b53bcf22e09916fb547d1ef82e84d3970472cca185ca2","pr_number":6807,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ba0d7b3cb993ff9033e32298aa0d95bdb64a8df4fef74fb7003e9ba9020bb775","failure_pattern":"context deadline exceeded. To troubleshoot, visit \u003curl\u003e","search_query":"context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0x17a8dd744af0?, {0x7f91c02f24c0?, 0x17a8dd261560?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1075 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0x17a8dcfdf200, {0x7f91c02f24c0, 0x17a8dd261560}, {0xf9f89cb, 0x15}, {0x17a8dc60e129, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:508 +0x2a8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func34.2({0x7f91c02f24c0, 0x17a8dd261560})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:68 +0x139\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x101a3e60?, 0x17a8dd261560?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 134\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xdfb\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked: failed to get subscription ID: ClientSecretCredential: unable to resolve an endpoint: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0x17a8dd744af0?, {0x7f91c02f24c0?, 0x17a8dd261560?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:1075 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0x17a8dcfdf200, {0x7f91c02f24c0, 0x17a8dd261560}, {0xf9f89cb, 0x15}, {0x17a8dc60e129, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:508 +0x2a8\ngithub.com/Azure/ARO-HCP/test/e2e.init.func34.2({0x7f91c02f24c0, 0x17a8dd261560})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:68 +0x139\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x101a3e60?, 0x17a8dd261560?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 134\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xdfb\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked: failed to get subscription ID: ClientSecretCredential: unable to resolve an endpoint: server response error:\n context deadline exceeded. To troubleshoot, visit https://aka.ms/azsdk/go/identity/troubleshoot#client-secret\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:476]: Test Panicked"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108229382379147264","occurred_at":"2026-10-08T16:13:48Z","signature_id":"c584d03b77f63107c8174575f5f10a9efc54b55c500b0c9908081f3f3d257f5a","pr_number":7232,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8d3193ed0a7e2c3492a5c226c9a0bdd4693921b8e35909048c2ef7aed3b7502d","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j4452224-mgmt-1\",\"clusterName\":\"ci01-j445222","search_query":"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T14:32:31.068088679Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T14:32:32.982339672Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T14:32:33.61885433Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-10-08T14:32:31.059Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-10-08T15:02:30.832Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-10-08T14:32:31.068088679Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\"}\\n{\\\"time\\\":\\\"2026-10-08T14:32:32.982339672Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-10-08T14:32:33.61885433Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4452224-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j4452224-mgmt-1\\\",\\\"region\\\":\\\"westus3\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7427/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108191395574452224","occurred_at":"2026-10-08T13:42:51Z","signature_id":"454655692b823febcffc54365e504c69961bb8ad6b414fb6c4922c06be0ce0d9","pr_number":7427,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dgrj4944q7jq\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":107,"prior_last_seen_at":"2026-10-04T23:52:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x255cb079f740\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dgrj4944q7jq\" to finish creating: GET https://rp.j5460992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a819b58-a3c9-46ef-aa03-af80037d2c41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a819b58-a3c9-46ef-aa03-af80037d2c41\",\n \"name\": \"5a819b58-a3c9-46ef-aa03-af80037d2c41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T15:30:05.316469426Z\",\n \"endTime\": \"2026-10-08T15:59:10.872777291Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dgrj4944q7jq--managed/providers/Microsoft.Compute/virtualMachines/l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dgrj4944q7jq--managed/providers/Microsoft.Compute/virtualMachines/l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x255cb079f740\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dgrj4944q7jq\" to finish creating: GET https://rp.j5460992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a819b58-a3c9-46ef-aa03-af80037d2c41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5a819b58-a3c9-46ef-aa03-af80037d2c41\",\n \"name\": \"5a819b58-a3c9-46ef-aa03-af80037d2c41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-08T15:30:05.316469426Z\",\n \"endTime\": \"2026-10-08T15:59:10.872777291Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dgrj4944q7jq--managed/providers/Microsoft.Compute/virtualMachines/l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dgrj4944q7jq--managed/providers/Microsoft.Compute/virtualMachines/l4c0q3f2g2d3k3l-np-delete-main-gqjhl-tns94 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7422/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108197422965460992","occurred_at":"2026-10-08T14:06:48Z","signature_id":"537d2f8688d12100b662592e26040e8091b73d73dada1ea5844df99344a5080b","pr_number":7422,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"e5bf9e749f8737ec65673ce5aaf89bc324878bae903ebb6ec2b0453ffece29b8","failure_pattern":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33","search_query":"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x119e8d352a00\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\n let _endTime = datetime(2026-10-08T14:29:33.7915314Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-08T14:29:42.5063608Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\", ActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", SubActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x119e8dac0f30\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x119e8d3529e0\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d3529c0\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d36b280\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d36b1a0\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x119e8b843000\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:96]: Timed out after 600.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x119e8d352a00\u003e: \n failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \"detailedInfraOrchestrationLogs\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\n let _endTime = datetime(2026-10-08T14:29:33.7915314Z);\n // Extract cluster ID from CS logs using cid field\n let _cs_cluster_id = toscalar(\n database('ServiceLogs').table('clustersServiceLogs')\n | where timestamp between (_startTime .. _endTime)\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\n | where isnotempty(log.cid)\n | distinct tostring(log.cid)\n | limit 1\n );\n // Find all Maestro resource IDs for this cluster\n let maestro_resource_ids = (\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | extend resourceId = extract(@'resourceid[=:\"]+([a-f0-9\\-]+)', 1, logStr)\n | where isnotempty(resourceId)\n | distinct resourceId\n );\n // Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\n union\n (\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\n // Includes: resource creation, updates, deletions, status events, gRPC communications\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"maestro\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\n // Includes the CR reconciliation logs\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"hypershift\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n ),\n (\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\n // Includes: work agent operations, manifest application, status feedback\n database('ServiceLogs').table('containerLogs')\n | where timestamp between (_startTime .. _endTime)\n | where namespace_name has \"open-cluster-management-agent\"\n | extend logStr = tostring(log)\n | where logStr has _cs_cluster_id\n | project timestamp, container_name, msg = logStr, log\n )\n | order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\"LimitsExceeded\", Message:\"Request is invalid and cannot be executed.\", Description:\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\", Type:\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\", Context:v2.ErrorContext{Timestamp:\"2026-10-08T14:29:42.5063608Z\", ServiceAlias:\"HCP-DEV-US-2\", MachineName:\"\", ProcessName:\"\", ProcessId:0, ThreadId:0, ClientRequestId:\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\", ActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", SubActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityType:\"GW.Http.CallContext\", ParentActivityId:\"21a4f26b-f20b-4845-825d-4d44325cd58e\", ActivityStack:\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\"}, IsPermanent:false})\n {\n msg: \"failed to gather logs: failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*errors.joinError | 0x119e8dac0f30\u003e{\n errs: [\n \u003c*fmt.wrapError | 0x119e8d3529e0\u003e{\n msg: \"failed to execute custom logs query: error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d3529c0\u003e{\n msg: \"error during query execution: failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d36b280\u003e{\n msg: \"failed to execute query: failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*fmt.wrapError | 0x119e8d36b1a0\u003e{\n msg: \"failed to read query \\\"detailedInfraOrchestrationLogs\\\" result (KQL: let _startTime = datetime(2026-10-07T14:29:33.7915312Z);\\nlet _endTime = datetime(2026-10-08T14:29:33.7915314Z);\\n// Extract cluster ID from CS logs using cid field\\nlet _cs_cluster_id = toscalar(\\n database('ServiceLogs').table('clustersServiceLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where resource_id has 'kusto-logs-7dxbsk584tbw'\\n | where isnotempty(log.cid)\\n | distinct tostring(log.cid)\\n | limit 1\\n);\\n// Find all Maestro resource IDs for this cluster\\nlet maestro_resource_ids = (\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | extend resourceId = extract(@'resourceid[=:\\\"]+([a-f0-9\\\\-]+)', 1, logStr)\\n | where isnotempty(resourceId)\\n | distinct resourceId\\n);\\n// Infrastructure \u0026 Orchestration Logs - Maestro, Hypershift, and ACM Agent\\nunion\\n (\\n // Maestro logs: Resource bundle operations for this cluster and its associated resources\\n // Includes: resource creation, updates, deletions, status events, gRPC communications\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"maestro\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id or logStr has_any (maestro_resource_ids)\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // Hypershift logs: Hypershift operator reconciliation of HostedCluster and NodePool CRs\\n // Includes the CR reconciliation logs\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"hypershift\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n ),\\n (\\n // ACM Agent logs: ManifestWork reconciliation on the managed cluster\\n // Includes: work agent operations, manifest application, status feedback\\n database('ServiceLogs').table('containerLogs')\\n | where timestamp between (_startTime .. _endTime)\\n | where namespace_name has \\\"open-cluster-management-agent\\\"\\n | extend logStr = tostring(log)\\n | where logStr has _cs_cluster_id\\n | project timestamp, container_name, msg = logStr, log\\n )\\n| order by timestamp asc): OneApiError(Error=v2.ErrorMessage{Code:\\\"LimitsExceeded\\\", Message:\\\"Request is invalid and cannot be executed.\\\", Description:\\\"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\\\", Type:\\\"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\\\", Context:v2.ErrorContext{Timestamp:\\\"2026-10-08T14:29:42.5063608Z\\\", ServiceAlias:\\\"HCP-DEV-US-2\\\", MachineName:\\\"\\\", ProcessName:\\\"\\\", ProcessId:0, ThreadId:0, ClientRequestId:\\\"KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666\\\", ActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", SubActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityType:\\\"GW.Http.CallContext\\\", ParentActivityId:\\\"21a4f26b-f20b-4845-825d-4d44325cd58e\\\", ActivityStack:\\\"(Activity stack: CRID=KGC.execute;2d601710-400e-4f38-8a97-e4a8f4407666 ARID=21a4f26b-f20b-4845-825d-4d44325cd58e \u003e GW.Http.CallContext/21a4f26b-f20b-4845-825d-4d44325cd58e)\\\"}, IsPermanent:false})\",\n err: \u003c*v2.OneApiError | 0x119e8b843000\u003e{\n ErrorMessage: {\n Code: \"LimitsExceeded\",\n Message: \"Request is invalid and cannot be executed.\",\n Description: \"Query execution has exceeded the allowed limits (80DA0003): The results of this query exceed the set limit of 64 MB, so not all records were returned (E_QUERY_RESULT_SET_TOO_LARGE, 0x80DA0003). See https://aka.ms/kustoquerylimits for more information and possible solutions..\",\n Type: \"Kusto.Data.Exceptions.KustoServicePartialQueryFailureLimitsExceededException\",\n Context: {\n Timestamp: ...,\n ServiceAlias: ...,\n MachineName: ...,\n ProcessName: ...,\n ProcessId: ...,\n ThreadId: ...,\n ClientRequestId: ...,\n ActivityId: ...,\n SubActivityId: ...,\n ActivityType: ...,\n ParentActivityId: ...,\n ActivityStack: ...,\n },\n IsPermanent: false,\n },\n },\n },\n },\n },\n },\n ],\n },\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7387/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108181128597737472","occurred_at":"2026-10-08T13:01:55Z","signature_id":"5b0ce0129345f929b9f05e55a452f600982aefa185554902a0b82bcf5372be48","pr_number":7387,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0984e2155380bc7af2966b58c7068460e7d241fdb82676269ebd409413b3d478","failure_pattern":"failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-22T13:01:50Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/prometheus","occurrences":1}],"full_error_samples":["time=2026-10-08T03:31:09.066Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus description=\"Step prometheus\\n Kind: Helm\\n\"\ntime=2026-10-08T03:31:09.688Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus values=\"map[environment:arohcpci01 kube-prometheus-stack:map[alertmanager:map[enabled:false] cleanPrometheusOperatorObjectNames:true coreDns:map[enabled:false] crds:map[enabled:true] fullnameOverride:prometheus global:map[rbac:map[create:true]] grafana:map[enabled:false] kube-state-metrics:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:kubernetes/kube-state-metrics sha:sha256:c5d026c4a8335c2d098ce04776e25babc3e7773e70817b5b709f388bb52488d2] metricLabelsAllowlist:[nodes=[node.kubernetes.io/instance-type,kubernetes.azure.com/agentpool]]] kubeApiServer:map[enabled:false] kubeControllerManager:map[enabled:false] kubeDns:map[enabled:false] kubeEtcd:map[enabled:false] kubeProxy:map[enabled:false] kubeScheduler:map[enabled:false] kubeStateMetrics:map[enabled:true] kubelet:map[enabled:false] nodeExporter:map[enabled:false] prometheus:map[enabled:false] prometheusOperator:map[admissionWebhooks:map[patch:map[enabled:true image:map[registry:arohcpsvcdev.azurecr.io repository:k8s-cache/ingress-nginx/kube-webhook-certgen sha:01038e7de14b78d702d2849c3aad72fd25903c4765af63cf16aa3398f5d5f2dd]]] affinity:map[nodeAffinity:map[requiredDuringSchedulingIgnoredDuringExecution:map[nodeSelectorTerms:[map[matchExpressions:[map[key:aro-hcp.azure.com/role operator:In values:[system]]]]]]]] enabled:true fullnameOverride: image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-operator sha:9c285c83d939df7d0bdf27287562bb6fcf8fd6ab639d899abba1f14a0c2d5647] prometheusConfigReloader:map[image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus-config-reloader sha:5fc5d09d0245cc1e7c6432ca9b0ebcc652c047b550d0289ea45725c14343585d]] thanosImage:map[registry:arohcpsvcdev.azurecr.io repository:thanos/thanos sha:b567818fe608067eb0f1d7c2c4fe361e7ad83c8a256234c97685f1d0bf670cc8 tag:v0.41.0] tolerations:[map[effect:NoSchedule key:infra operator:Equal value:true]]]] prometheus:map[serviceAccount:map[managedIdentity:0b00177b-a54e-4160-b95f-d8ba55b89f42]] prometheusSpec:map[externalLabels:map[cluster:ci01-j0275840-svc environment:ci01 region:westus3] hcpRemoteWriteUrl:NONE image:map[registry:mcr.microsoft.com/oss/v2 repository:prometheus/prometheus sha:7ef9c1d80e4e8e1eec1a207166af22930c8f11200f25001d1a3285b30b4f13f9] maximumStartupDurationSeconds:360 remoteWriteUrl:https://msprom-westus3-ci01-j0275840-svc-ng1v.westus3-1.metrics.ingest.monitor.azure.com/dataCollectionRules/dcr-d00da46377ca403ab27fbe4e13d51efe/streams/Microsoft-PrometheusMetrics/api/v1/write?api-version=2023-04-24 replicas:2 resources:map[limits:map[memory:NONE] requests:map[cpu:NONE memory:NONE]] shards:1 version: zoneCount:3] schedulingRole:system]\"\ntime=2026-10-08T03:31:09.688Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-10-08T03:31:10.214Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus namespace=prometheus\ntime=2026-10-08T03:31:10.584Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus dryRun=false\ntime=2026-10-08T03:31:10.584Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-10-08T03:31:10.783Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus\ntime=2026-10-08T03:40:20.024Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-10-08T03:40:20.024Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=prometheus err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed pre-install: resource Job/prometheus/prometheus-admission-create not ready. status: Failed, message: Job Failed. failed: 1/1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108026812570275840","occurred_at":"2026-10-08T02:48:51Z","signature_id":"6bac7c9167143bb4a1dfabfc4a6d81083bef0f715dcabad843cd680ef59999d4","pr_number":6775,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.098901098901099,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-02..2026-10-08 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":17,"prior_last_seen_at":"2026-09-22T06:52:43Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","occurrences":1}],"full_error_samples":["time=2026-10-08T14:39:49.369Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-08T14:39:51.193Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-08T14:39:53.050Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=4570d5430c346685aef785faa645617cca8a23038712be321dc348c5ac428645 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1446656-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4570d5430c346685aef785faa645617cca8a23038712be321dc348c5ac428645\ntime=2026-10-08T15:09:49.053Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7414/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2108199393621446656","occurred_at":"2026-10-08T14:14:35Z","signature_id":"546713f253dbb07bdf6d6b240d468c73b1927023c4bbf21dee04e87aaa304e9a","pr_number":7414,"after_last_push_of_merged_pr":true}]}]}]}