{"meta":{"start_date":"2026-10-23","end_date":"2026-10-23","start_at":"2026-10-23T00:00:00Z","end_at":"2026-10-24T00:00:00Z","timezone":"UTC","generated_at":"2026-10-30T02:08:24Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":78,"failed_runs":53,"raw_occurrences":112,"matched_occurrences":106,"runs_affected":47},"rows":[{"environment":"dev","failure_pattern_id":"886ca4073dc118fed59b69c1c77c2f874c2ef49c4467f17e6fcb9bd2d07448e6","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":12,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":12,"after_last_push_count":0,"trend_counts":[0,0,0,1,1,0,12],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x33f93d769890\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x33f93d769890\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x327f918b1650\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x327f918b1650\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2fa2959cc2d0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2fa2959cc2d0\u003e: \n ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n {\n s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"06077bac2bdfbefc430aa21e7f1c63585e4b0facfbed8535434aa7846c098a94","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"150ce84edc5920033c79f1d71e7933469c3d65ffd64a306a05fe2dabbc654d50","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"5d631566a7a8e50a23a29e16a0e18527e365de9f11bacc81791b4a3fa0dd48df","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"6a69757f3cbee730cf9d60087714cca23dc702792cdaa339a7014d9b67fd1143","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"70670de7f9371c033d82071ca8d86a5ee534cd8acfcecfb1faac595d80a44997","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"712adee26418d9b97d2614e8f26ca63599b26688a67ad28c5cce4a2daa4c4e84","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"905453ac6e335acb4f35cd240fa3b7536659d0d95ac9186a5592efab0f975eea","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"99fcfb8a0f2194c211d91e98d4874fd0b7609259139ff0a43afeb62267c42148","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"b79149b9fc941d47e42641fbc00f500cd063ac6efa35986bbc24fa7b5d878613","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"bb412fd2d485854280b1c9d07b101b3e663f2d9f084fff735671240db5800f4f","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"d879674894167409266496657d54be06c8aa7f724945913ba06a1ba076ac2290","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"f75c1d43c7a4c5c09304ce65eeb484f3c9012f635c820f63f75d2054e572fc61","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f48bqq4pfgdf\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":7.6923076923076925,"anchor_occurrences":6,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,6],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":146,"prior_last_seen_at":"2026-10-16T16:35:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x26e13640a140\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-jft4smjjlxxw\" to finish creating: GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\",\n \"name\": \"fb71e438-88d3-4b63-82a0-b820e717c220\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T00:13:46.41509851Z\",\n \"endTime\": \"2026-10-24T00:32:46.057566699Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\n correlation ID: 97b55a4e-bfeb-47d6-99bd-fc35a2cc6359\n client request ID: c7f67463-49ac-4db2-9fc4-28f026c30acf\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x26e13640a140\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-jft4smjjlxxw\" to finish creating: GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\",\n \"name\": \"fb71e438-88d3-4b63-82a0-b820e717c220\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-24T00:13:46.41509851Z\",\n \"endTime\": \"2026-10-24T00:32:46.057566699Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8222464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb71e438-88d3-4b63-82a0-b820e717c220\n correlation ID: 97b55a4e-bfeb-47d6-99bd-fc35a2cc6359\n client request ID: c7f67463-49ac-4db2-9fc4-28f026c30acf","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xa03dff15100\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-k98p95t7rwlk\" to finish creating: GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\",\n \"name\": \"1937a216-e9d1-46bb-b9f8-a55a34e91da8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T20:32:20.327838457Z\",\n \"endTime\": \"2026-10-23T20:51:29.045056606Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2p298-8vgqx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2p298-8vgqx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\n correlation ID: f3bdc38e-f649-43a0-824c-a60a987dceb4\n client request ID: f8924ec9-c563-43bf-b506-67458e3fbd1f\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xa03dff15100\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-k98p95t7rwlk\" to finish creating: GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\",\n \"name\": \"1937a216-e9d1-46bb-b9f8-a55a34e91da8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T20:32:20.327838457Z\",\n \"endTime\": \"2026-10-23T20:51:29.045056606Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2p298-8vgqx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2p298-8vgqx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-k98p95t7rwlk--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2p298-8vgqx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3087360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1937a216-e9d1-46bb-b9f8-a55a34e91da8\n correlation ID: f3bdc38e-f649-43a0-824c-a60a987dceb4\n client request ID: f8924ec9-c563-43bf-b506-67458e3fbd1f","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x21c5dc2ca7e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f88rfw7wtj4v\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\",\n \"name\": \"b02b5a9b-3f7f-467b-bc65-f56966e21cc5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:32:08.161782793Z\",\n \"endTime\": \"2026-10-23T18:51:14.083013713Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-f88rfw7wtj4v--managed/ingress-cert-le-np-1-b4g5h-9bn6x-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-f88rfw7wtj4v--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-b4g5h-9bn6x-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\n correlation ID: 053e3b77-21ef-4381-b997-ce492e16aa00\n client request ID: d87da6eb-f6e0-450c-b35a-d8d6ce034e64\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:165]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x21c5dc2ca7e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-f88rfw7wtj4v\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\",\n \"name\": \"b02b5a9b-3f7f-467b-bc65-f56966e21cc5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:32:08.161782793Z\",\n \"endTime\": \"2026-10-23T18:51:14.083013713Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-b4g5h-9bn6x: Failed: interfaces failed to create or update. err: failed to get existing resource ingress-cert-le-f88rfw7wtj4v--managed/ingress-cert-le-np-1-b4g5h-9bn6x-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-f88rfw7wtj4v--managed/providers/Microsoft.Network/networkInterfaces/ingress-cert-le-np-1-b4g5h-9bn6x-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '10' seconds after additional tokens are available. Refer ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b02b5a9b-3f7f-467b-bc65-f56966e21cc5\n correlation ID: 053e3b77-21ef-4381-b997-ce492e16aa00\n client request ID: d87da6eb-f6e0-450c-b35a-d8d6ce034e64"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7736/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113768338508222464","occurred_at":"2026-10-23T23:03:38Z","signature_id":"186c2f62dbba5354fdd50e3f198e2f5c7260e536efa09a9dba4dd236da4ad630","pr_number":7736,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113714121043087360","occurred_at":"2026-10-23T19:28:05Z","signature_id":"bb2f64cac1081e7e6bb8f566f3bd4023215ffc8e942c129ba96fe59be1320761","pr_number":7722,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680800309317632","occurred_at":"2026-10-23T17:15:47Z","signature_id":"edc4a2a86804f16bb2ca5ef5fec66418b27ca460d596142f26648f4d317f99da","pr_number":7657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113660833035718656","occurred_at":"2026-10-23T15:56:27Z","signature_id":"d9f385324d27a77fd6938089006a621965c3093828acb8f05519d964d111d9ed","pr_number":7073,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7658/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113542145401950208","occurred_at":"2026-10-23T08:02:16Z","signature_id":"d04d5f67613fcded3e78320b3bd7ce7a8186accddad999333f00e221da713c03","pr_number":7658,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"5efae9732adc28512b06dafb9969fee7b2e6984fa695d99435471c1675c6ced2","pr_number":7463,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"ed775075c8b32bf09c1ff72fb864563b60eeb162dc88cdf2a9311ae63c4f318c","failure_pattern":"failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=1)","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":7.6923076923076925,"anchor_occurrences":6,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,6],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=1)","occurrences":3},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/management/deploy (stamp=2)","occurrences":3}],"full_error_samples":["time=2026-10-24T00:10:57.890Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-24T00:10:58.200Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-24T00:10:58.200Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-24T00:10:58.456Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-24T00:10:58.679Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-24T00:10:58.679Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-24T00:10:58.736Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-24T00:16:02.844Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-24T00:16:02.844Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","time=2026-10-23T15:33:31.685Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-23T15:33:32.582Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-23T15:33:32.582Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T15:33:33.039Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 namespace=secret-sync-controller\ntime=2026-10-23T15:33:33.396Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 dryRun=false\ntime=2026-10-23T15:33:33.396Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T15:33:33.530Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1\ntime=2026-10-23T15:38:39.422Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-23T15:38:39.423Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","time=2026-10-23T12:34:43.631Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-23T12:34:44.030Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 values=\"map[controllerName:secrets-store-sync-controller-manager image:arohcpsvcdev.azurecr.io/secrets-store-sync/controller@sha256:b98f9f1c3c603f0e3ee70cd3d70746d8265b8d425df5b71f23c4b0dcb3eb979c imagePullPolicy:IfNotPresent metricsPort:8085 providerContainer:[map[args:[--endpoint=unix:///provider/azure.sock --construct-pem-chain=true --healthz-port=8989 --healthz-path=/healthz --healthz-timeout=5s] image:mcr.microsoft.com/oss/v2/azure/secrets-store/provider-azure@sha256:b451542dd17a9b1b1b400fa1ca768a492720b87b696f373f0494217d855aee8c imagePullPolicy:IfNotPresent livenessProbe:map[failureThreshold:3 httpGet:map[path:/healthz port:8989] initialDelaySeconds:5 periodSeconds:30 timeoutSeconds:10] name:provider-azure-installer ports:[map[containerPort:8898 name:metrics protocol:TCP]] resources:map[limits:map[memory:unlimited] requests:map[cpu:50m memory:100Mi]] securityContext:map[allowPrivilegeEscalation:false capabilities:map[drop:[ALL]] readOnlyRootFilesystem:true runAsUser:0] volumeMounts:[map[mountPath:/provider name:providervol]]]] tokenRequestAudience:[map[audience:\u003cnil\u003e]] validatingAdmissionPolicies:map[allowedSecretTypes:[Opaque kubernetes.io/basic-auth bootstrap.kubernetes.io/token kubernetes.io/dockerconfigjson kubernetes.io/dockercfg kubernetes.io/ssh-auth kubernetes.io/tls] applyPolicies:true deniedSecretTypes:[kubernetes.io/service-account-token]]]\"\ntime=2026-10-23T12:34:44.030Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-23T12:34:44.168Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 namespace=secret-sync-controller\ntime=2026-10-23T12:34:44.296Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-10-23T12:34:44.296Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-23T12:34:44.405Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2\ntime=2026-10-23T12:39:47.576Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-23T12:39:47.576Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/secret-sync-controller/secrets-store-sync-controller-manager not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113771192396550144","occurred_at":"2026-10-23T23:14:58Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113646623203528704","occurred_at":"2026-10-23T14:59:59Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113597591521333248","occurred_at":"2026-10-23T11:45:03Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113544934832214016","occurred_at":"2026-10-23T08:15:49Z","signature_id":"a08e9c6d28fcde6c9a776a1ba68e907e93cf331a6c9e832a31bd1ee1c3f67256","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113446671063977984","occurred_at":"2026-10-23T01:45:26Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113432151310471168","occurred_at":"2026-10-23T00:47:45Z","signature_id":"2294386d02fe4b81413deb651032e91a325a0021a4c26d54153e4b7cb251ee5d","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.8461538461538463,"anchor_occurrences":5,"after_last_push_count":14,"trend_counts":[0,0,2,6,2,0,5],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":2,"prior_window_starts":["2026-10-05","2026-10-12"],"prior_runs_affected":31,"prior_last_seen_at":"2026-10-16T13:20:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3dd71ecc7860\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-gvwnv545c9xf\" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\",\n \"name\": \"b77ed110-707e-4c38-ab0a-33ad172ee68c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T21:19:37.486805843Z\",\n \"endTime\": \"2026-10-23T21:38:45.647369611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\n correlation ID: 6572f7b9-3e5a-422f-aa65-926c920bbad1\n client request ID: bb39b90a-4ada-4996-91b9-6bd38bf61535\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3dd71ecc7860\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-gvwnv545c9xf\" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\",\n \"name\": \"b77ed110-707e-4c38-ab0a-33ad172ee68c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T21:19:37.486805843Z\",\n \"endTime\": \"2026-10-23T21:38:45.647369611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b77ed110-707e-4c38-ab0a-33ad172ee68c\n correlation ID: 6572f7b9-3e5a-422f-aa65-926c920bbad1\n client request ID: bb39b90a-4ada-4996-91b9-6bd38bf61535\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:328]: failed to create worker nodepool for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x204486f8560\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-wrxzvkddb94j\" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\",\n \"name\": \"13760d44-2d42-4c53-9fca-8d97381d23a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T21:19:36.261324147Z\",\n \"endTime\": \"2026-10-23T21:38:45.545540127Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\n correlation ID: 5d55a2cb-56aa-47f7-a5b9-cf7b9a27d226\n client request ID: f043a8e7-f2d1-473c-b2e3-11abda25efb3\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:328]: failed to create worker nodepool for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x204486f8560\u003e: \n failed to create NodePool worker: failed waiting for nodepool=\"worker\" for cluster \"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-wrxzvkddb94j\" to finish creating: GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\",\n \"name\": \"13760d44-2d42-4c53-9fca-8d97381d23a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T21:19:36.261324147Z\",\n \"endTime\": \"2026-10-23T21:38:45.545540127Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1278720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13760d44-2d42-4c53-9fca-8d97381d23a5\n correlation ID: 5d55a2cb-56aa-47f7-a5b9-cf7b9a27d226\n client request ID: f043a8e7-f2d1-473c-b2e3-11abda25efb3\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-sn4xdl\" for cluster \"cluster-candidate-4-21-sn4xdl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x60436518ee0\u003e: \n failed to create NodePool np-sn4xdl: failed waiting for nodepool=\"np-sn4xdl\" for cluster \"cluster-candidate-4-21-sn4xdl\" in resourcegroup=\"rg-candidate-4-21-c4tsdrwbnnfb\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\",\n \"name\": \"d83a0f27-909b-4319-9bb2-43e65d178f6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:34:43.074031046Z\",\n \"endTime\": \"2026-10-23T18:53:43.987521771Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\n correlation ID: 327dae19-27b0-4a9b-8e18-5ab3841e0ec7\n client request ID: 2803ecfb-10a6-4276-a6a2-935c10a7ca77\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-sn4xdl\" for cluster \"cluster-candidate-4-21-sn4xdl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x60436518ee0\u003e: \n failed to create NodePool np-sn4xdl: failed waiting for nodepool=\"np-sn4xdl\" for cluster \"cluster-candidate-4-21-sn4xdl\" in resourcegroup=\"rg-candidate-4-21-c4tsdrwbnnfb\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\",\n \"name\": \"d83a0f27-909b-4319-9bb2-43e65d178f6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:34:43.074031046Z\",\n \"endTime\": \"2026-10-23T18:53:43.987521771Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d83a0f27-909b-4319-9bb2-43e65d178f6e\n correlation ID: 327dae19-27b0-4a9b-8e18-5ab3841e0ec7\n client request ID: 2803ecfb-10a6-4276-a6a2-935c10a7ca77\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113725096341278720","occurred_at":"2026-10-23T20:11:48Z","signature_id":"96d11e5d24c2f96b51ede935923d2da2885cc5293b4b2067a83db979216fe98e","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113725096341278720","occurred_at":"2026-10-23T20:11:48Z","signature_id":"e4cd001bc94a7c7f7d880d8bd2124ba820cefb9f6079b7e26d5709adbf7dc519","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680800309317632","occurred_at":"2026-10-23T17:15:47Z","signature_id":"30e7ab70c0561d3688101981f498a87ca5fd5787330e0b440c1bbb06abf887df","pr_number":7657,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113420718787006464","occurred_at":"2026-10-23T00:01:14Z","signature_id":"4fa3b47449004f479ac51d54be8b694777e1df03e185828cb11995c7ea0071e0","pr_number":7688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113420718787006464","occurred_at":"2026-10-23T00:01:14Z","signature_id":"6579f8d0a151b49461fd52fcea9ac6cb7fc7ffd497fe8e411acf99fb8274cfa3","pr_number":7688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bfa0d868a12599cc0f37a473c69f092416803c7127cd792bfcc6040d8e41c230","failure_pattern":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":5.128205128205129,"anchor_occurrences":4,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,4],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":4}],"full_error_samples":["time=2026-10-24T00:11:13.169Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-24T00:11:14.722Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-24T00:11:14.722Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-24T00:11:15.608Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-24T00:11:15.608Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:11:15.609Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j2212352-svc\ntime=2026-10-24T00:13:20.091Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-24T00:13:20.091Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=2m4s\ntime=2026-10-24T00:13:20.091Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.407Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-24T00:13:20.407Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.407Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.862Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-24T00:13:20.862Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:20.894Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-24T00:13:21.097Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.168Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j2212352-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-24T00:13:21.740Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j2212352-svc\ntime=2026-10-24T00:28:24.267Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\"","time=2026-10-23T22:20:23.920Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-23T22:20:25.642Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-23T22:20:25.642Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-23T22:20:26.982Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-23T22:20:26.982Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:20:26.982Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j9780224-svc\ntime=2026-10-23T22:23:03.498Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-23T22:23:03.499Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=2m37s\ntime=2026-10-23T22:23:03.499Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:04.054Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-23T22:23:04.054Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:04.054Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:04.811Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-23T22:23:04.811Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:04.874Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-23T22:23:05.250Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:05.375Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:06.263Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:06.263Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:06.263Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:06.263Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j9780224-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T22:23:06.263Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j9780224-svc\ntime=2026-10-23T22:38:09.485Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\"","time=2026-10-23T18:08:55.483Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-23T18:08:56.785Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-23T18:08:56.785Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-23T18:08:57.580Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-23T18:08:57.580Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:08:57.580Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j6973824-svc\ntime=2026-10-23T18:13:02.175Z level=INFO msg=\"Cluster provisioning state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade state=Succeeded\ntime=2026-10-23T18:13:02.175Z level=INFO msg=\"ARM operation complete\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary elapsed=4m5s\ntime=2026-10-23T18:13:02.175Z level=INFO msg=\"Step 1/9: Ensuring MISE ext-authz ConfigMap\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:02.795Z level=INFO msg=\"ConfigMap created\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade/revision-configmap name=istio-shared-configmap-asm-1-30\ntime=2026-10-23T18:13:02.795Z level=INFO msg=\"Step 2/9: Checking tag-based namespace safety\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:02.795Z level=INFO msg=\"Step 3/9: Verifying control plane health and flipping tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:03.759Z level=INFO msg=\"Created revision tag webhook\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade/revision-tag webhook=istio-revision-tag-prod-stable-aks-istio-system revision=asm-1-30\ntime=2026-10-23T18:13:03.759Z level=INFO msg=\"Step 4/9: Ensuring ingress gateway annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:03.939Z level=INFO msg=\"Applying annotations\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade/ingress-annotations service=aks-istio-ingressgateway-external\ntime=2026-10-23T18:13:04.167Z level=INFO msg=\"Step 5/9: Migrating workloads to target revision\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.333Z level=INFO msg=\"Step 6/9: Running health check\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.935Z level=INFO msg=\"Health check passed -- checking for orphaned workloads before completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.935Z level=INFO msg=\"Step 7/9: Running orphan guard\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.935Z level=INFO msg=\"No orphaned workloads -- completing canary\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.935Z level=INFO msg=\"Step 8/9: Completing canary via ARM (point of no return)\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j6973824-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T18:13:04.935Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=complete-canary cluster=ci01-j6973824-svc\ntime=2026-10-23T18:28:07.753Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to complete canary: complete-canary: ARM operation timed out or failed: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7120/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113774695802212352","occurred_at":"2026-10-23T23:28:54Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7120,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7120/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113750398169780224","occurred_at":"2026-10-23T21:52:21Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7120,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113683432016973824","occurred_at":"2026-10-23T17:26:03Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113627585765183488","occurred_at":"2026-10-23T13:44:20Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.8461538461538463,"anchor_occurrences":3,"after_last_push_count":36,"trend_counts":[0,0,4,9,1,3,3],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":108,"prior_last_seen_at":"2026-10-15T19:10:04Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":3}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T20:05:44Z\n Ended: 2026-10-23T20:17:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T20:05:45Z\n Ended: 2026-10-23T20:17:41Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T18:28:28Z\n Ended: 2026-10-23T18:40:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T18:28:28Z\n Ended: 2026-10-23T18:40:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T18:30:05Z\n Ended: 2026-10-23T18:41:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9103104-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.24:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-t567t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T18:30:05Z\n Ended: 2026-10-23T18:42:04Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9103104-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.24:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-t567t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-23T18:31:05Z\n Ended: 2026-10-23T18:40:13Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9103104-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.24:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-t567t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-23T18:31:05Z\n Ended: 2026-10-23T18:40:13Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9103104-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"kube-apiserver\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.24:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9103104/providers/microsoft.monitor/accounts/hcps-j9103104\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-t567t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2tkl2i5v870ibn1t5qn7lc02tk2a81jm-l4a0x1d2r1d3t9r/kube-apiserver has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"2286281296e2588a0e23ff286161d911dea899212cf719d1a69d41fc8aea0670","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","signature_id":"b9051e9b8ccf6adbe45aedab22f7090fbb17021fbbdd41caae75f1e8b7eda5f2","pr_number":7722,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","signature_id":"56a88ed15c81d66a2d6b20e1497719151538162e99edd29f52fd5232a43bb042","pr_number":7547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.8461538461538463,"anchor_occurrences":3,"after_last_push_count":20,"trend_counts":[0,0,5,8,0,2,3],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":59,"prior_last_seen_at":"2026-10-15T16:20:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":3}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T19:53:47Z\n Ended: 2026-10-23T20:19:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ad470ce0-381a-4372-b725-f995a508a24f\"\n Description: Pod ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-23T19:53:47Z\n Ended: 2026-10-23T20:19:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ad470ce0-381a-4372-b725-f995a508a24f\"\n Description: Pod ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T18:17:28Z\n Ended: 2026-10-23T18:44:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9b5be082-3743-4901-bdab-8d87701f28f1\"\n Description: Pod ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-23T18:17:28Z\n Ended: 2026-10-23T18:44:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9b5be082-3743-4901-bdab-8d87701f28f1\"\n Description: Pod ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T06:49:50Z\n Ended: 2026-10-23T06:56:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3270400-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.213:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3270400/providers/microsoft.monitor/accounts/hcps-j3270400\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le\", pod=\"capi-provider-7d64b7b878-2mwb4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"40ed8520-738a-4471-a9a5-0fd0a1d23770\"\n Description: Pod ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le/capi-provider-7d64b7b878-2mwb4 (manager) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-23T06:49:50Z\n Ended: 2026-10-23T06:56:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3270400-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.213:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3270400/providers/microsoft.monitor/accounts/hcps-j3270400\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le\", pod=\"capi-provider-7d64b7b878-2mwb4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"40ed8520-738a-4471-a9a5-0fd0a1d23770\"\n Description: Pod ocm-arohcpci01-2tkaqmvd2oh7rag7g43l8hppd6nmg8t0-ingress-cert-le/capi-provider-7d64b7b878-2mwb4 (manager) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"5f39e806b7aca5a0dc9a2e9d8787423dbc4725456fa3be8c785dbeffe0c743c5","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","signature_id":"ed02ebe0267da497eb0e3be41998c6fcb596d452363ba5bd7181d595e7764413","pr_number":7722,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"19ad7954ab7f29b25a54f19b9d0544d2410dbddc79a38cc95515845245055d71","pr_number":7463,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.8461538461538463,"anchor_occurrences":3,"after_last_push_count":41,"trend_counts":[1,2,13,8,11,4,3],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":154,"prior_last_seen_at":"2026-10-18T21:09:39Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T15:27:17Z\n Ended: 2026-10-23T15:33:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5062656-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tkibdc5gs5nul5ksrupnsegrco0ubpn-w6p5u2c0q5q4w4r\", pod=\"ignition-server-proxy-bf7d94666-n9xlf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tkibdc5gs5nul5ksrupnsegrco0ubpn-w6p5u2c0q5q4w4r/ignition-server-proxy-bf7d94666-n9xlf has been in a non-ready state for longer than 5 minutes.","alert fired 11 time(s) (3 unknown, 8 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T13:16:41Z\n Ended: 2026-10-23T13:26:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6471040-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s\", pod=\"cluster-network-operator-6857fbf8d7-5cgmz\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s/cluster-network-operator-6857fbf8d7-5cgmz has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T13:16:41Z\n Ended: 2026-10-23T13:26:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6471040-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s\", pod=\"ingress-operator-787469b994-cfnzc\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s/ingress-operator-787469b994-cfnzc has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-23T13:16:41Z\n Ended: 2026-10-23T13:26:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6471040-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s\", pod=\"dns-operator-6d4899cf84-7qfzv\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tkg8fusv1drnq89nr4jr4kc118h003a-d8f2g8w6t8m3b5s/dns-operator-6d4899cf84-7qfzv has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T03:34:48Z\n Ended: 2026-10-23T03:58:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2625664-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tk89mvlqfc5cf20qag1guihinhj7c0v-z6o9i3i9h0a8n5p\", pod=\"router-5cfd955d57-tpd7s\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2tk89mvlqfc5cf20qag1guihinhj7c0v-z6o9i3i9h0a8n5p/router-5cfd955d57-tpd7s has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7717/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113628897605062656","occurred_at":"2026-10-23T13:49:33Z","signature_id":"762381bb4f426bcce14d2dcedf2af2dc5db6afa5d64699c2dabcc9907a483ca0","pr_number":7717,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"6fbf7b7865ca254c8d602bde28984e80b575972637dde5ddb43c09fae006779f","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113459428752625664","occurred_at":"2026-10-23T02:36:08Z","signature_id":"b85e109c2f4dc5d74b9ee622dcf47c821f416845163d08b78b0f76a24ed1a9fa","pr_number":7687,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":3.8461538461538463,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,0,0,2,6,3],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":11,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","occurrences":3}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T22:38:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9171712-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-cqcsl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"24c5a63c-ba67-481c-97bd-1b2284e4616c\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-cqcsl (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T22:40:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9171712-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.104:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-cqcsl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"24c5a63c-ba67-481c-97bd-1b2284e4616c\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-cqcsl (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-23T22:41:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9171712-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-l6h2t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"dc5bdfb6-fff6-4941-9c02-12766a972bae\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-l6h2t (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-23T22:43:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9171712-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.221:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9171712/providers/microsoft.monitor/accounts/services-j9171712\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-l6h2t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"dc5bdfb6-fff6-4941-9c02-12766a972bae\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-l6h2t (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T19:45:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-wpx95\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f99f4808-64ca-4f14-b866-30cd9d5a44aa\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-wpx95 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T19:48:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-b4nrw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b83bb111-3a9c-4b96-b2ca-76053771e1a5\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-b4nrw (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-23T19:48:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-b4nrw\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b83bb111-3a9c-4b96-b2ca-76053771e1a5\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-b4nrw (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-23T19:51:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/services-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-wpx95\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f99f4808-64ca-4f14-b866-30cd9d5a44aa\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-wpx95 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T12:35:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-558mc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a45ad8a2-0299-48d6-b5cc-feba766709c3\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-558mc (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T12:38:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0830592-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.214:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-9dtps\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a3569b9f-561d-4f48-9ca7-6bcdc31d1728\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-9dtps (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-23T12:38:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0830592-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.214:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-9dtps\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a3569b9f-561d-4f48-9ca7-6bcdc31d1728\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-9dtps (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-23T12:40:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-7c96c6b89d-558mc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"a45ad8a2-0299-48d6-b5cc-feba766709c3\"\n Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-7c96c6b89d-558mc (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113744756189171712","occurred_at":"2026-10-23T21:29:55Z","signature_id":"b6d85dbc1d4502551493043fa33721464a3b1282465557d679ae70789ff0b520","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"2ec5e8a05ec938e5d3691a4dec2f741db485c232fb0b69893230211b7263938c","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","signature_id":"fd738f0b860459440540292342939e3ce110921d918faf32d78e2b8f0cbc73ec","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"392750d533ddeabf76d5209f9e2a09e5f48e64ee98fce8c13f1c6bd36af29cfa","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-05"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-09T21:37:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["src/runtime/mgc.go:1695 +0x17 fp=0xccc45f767e0 sp=0xccc45f767c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f767e8 sp=0xccc45f767e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0xccc46384000 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638a740 sp=0xccc4638a720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638a7c8 sp=0xccc4638a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638a7e0 sp=0xccc4638a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638a7e8 sp=0xccc4638a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0xccc461ac3c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f7d740 sp=0xccc45f7d720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f7d7c8 sp=0xccc45f7d740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7d7e0 sp=0xccc45f7d7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7d7e8 sp=0xccc45f7d7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0xccc460045a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f76f40 sp=0xccc45f76f20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f76fc8 sp=0xccc45f76f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f76fe0 sp=0xccc45f76fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f76fe8 sp=0xccc45f76fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0xccc463841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638af40 sp=0xccc4638af20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638afc8 sp=0xccc4638af40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638afe0 sp=0xccc4638afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638afe8 sp=0xccc4638afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0xccc461ac5a0 m=5 mp=0xccc46000008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0xccc45f7dea0 sp=0xccc45f7de90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0xccc45f7df40 sp=0xccc45f7dea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0xccc45f7dfc8 sp=0xccc45f7df40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7dfe0 sp=0xccc45f7dfc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7dfe8 sp=0xccc45f7dfe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 36 gp=0xccc463843c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207e7eb6?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638b740 sp=0xccc4638b720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638b7c8 sp=0xccc4638b740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638b7e0 sp=0xccc4638b7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638b7e8 sp=0xccc4638b7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax 0x0\nrbx 0x7fc7134fd640\nrcx 0x7fc75a9bf94c\nrdx 0x6\nrdi 0xa70a\nrsi 0xa711\nrbp 0xa711\nrsp 0x7fc7134fc790\nr8 0x7fc7134fc860\nr9 0x7fc75aae54e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7fc7134fcb60\nr14 0xccc45e750e0\nr15 0xffffffffffffffff\nrip 0x7fc75a9bf94c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\nsrc/runtime/mgc.go:1695 +0x17 fp=0xccc45f767e0 sp=0xccc45f767c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f767e8 sp=0xccc45f767e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0xccc46384000 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638a740 sp=0xccc4638a720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638a7c8 sp=0xccc4638a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638a7e0 sp=0xccc4638a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638a7e8 sp=0xccc4638a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0xccc461ac3c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f7d740 sp=0xccc45f7d720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f7d7c8 sp=0xccc45f7d740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7d7e0 sp=0xccc45f7d7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7d7e8 sp=0xccc45f7d7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0xccc460045a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc45f76f40 sp=0xccc45f76f20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc45f76fc8 sp=0xccc45f76f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f76fe0 sp=0xccc45f76fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f76fe8 sp=0xccc45f76fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0xccc463841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638af40 sp=0xccc4638af20 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638afc8 sp=0xccc4638af40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638afe0 sp=0xccc4638afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638afe8 sp=0xccc4638afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0xccc461ac5a0 m=5 mp=0xccc46000008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0xccc45f7dea0 sp=0xccc45f7de90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0xccc45f7df40 sp=0xccc45f7dea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0xccc45f7dfc8 sp=0xccc45f7df40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc45f7dfe0 sp=0xccc45f7dfc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc45f7dfe8 sp=0xccc45f7dfe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 36 gp=0xccc463843c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207e7eb6?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0xccc4638b740 sp=0xccc4638b720 pc=0x566028e\nruntime.gcBgMarkWorker(0xccc45f463f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0xccc4638b7c8 sp=0xccc4638b740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0xccc4638b7e0 sp=0xccc4638b7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0xccc4638b7e8 sp=0xccc4638b7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax 0x0\nrbx 0x7fc7134fd640\nrcx 0x7fc75a9bf94c\nrdx 0x6\nrdi 0xa70a\nrsi 0xa711\nrbp 0xa711\nrsp 0x7fc7134fc790\nr8 0x7fc7134fc860\nr9 0x7fc75aae54e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7fc7134fcb60\nr14 0xccc45e750e0\nr15 0xffffffffffffffff\nrip 0x7fc75a9bf94c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","7f52 bbf59cb1 ...nR.......R...\n00007ffd4f7fae70: 00007f52 6e7fc640 00007ffd 4f7fb140 @..nR...@..O....\n00007ffd4f7fae80: 00007ffd 4f7fafae 00007ffd 4f7fafaf ...O.......O....\n00007ffd4f7fae90: 00007f52 6e7fc640 00007f52 bbed47f5 @..nR....G..R...\n00007ffd4f7faea0: 00007f52 bc045c80 00000000 00000120 .\\..R... .......\n00007ffd4f7faeb0: 00000000 003d0f00 b406abf0 07e46d00 ..=......m......\n00007ffd4f7faec0: 00007f52 bbe48740 00000000 00000006 @...R...........\n00007ffd4f7faed0: 00000000 1a0a9f00 00007ffd 4f7fb200 ...........O....\n00007ffd4f7faee0: 00001a1d 095421e0 00007f52 bbe89646 .!T.....F...R...\n00007ffd4f7faef0: 00007f52 bc046e90 00007f52 bbe737f3 .n..R....7..R...\n00007ffd4f7faf00: 00000000 00000020 b406abf0 07e46d00 ........m......\n00007ffd4f7faf10: 00007f52 6dffc000 00007f52 bbf48ac4 ...mR.......R...\n00007ffd4f7faf20: 00000000 00000000 00000000 0000000d ................\n\ngoroutine 1 gp=0x1a1d095421e0 m=0 mp=0x17528da0 [running, locked to thread]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x1a1d096486a8 sp=0x1a1d09648698 pc=0x56668c8\nruntime.newm1(0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:2922 +0x85 fp=0x1a1d096486e8 sp=0x1a1d096486a8 pc=0x562b625\nruntime.newm(0x5635301?, 0x0, 0xd19a3c0?)\n\t/usr/lib/golang/src/runtime/proc.go:2902 +0x125 fp=0x1a1d09648718 sp=0x1a1d096486e8 pc=0x562b525\nruntime.startTemplateThread()\n\t/usr/lib/golang/src/runtime/proc.go:2947 +0x85 fp=0x1a1d09648748 sp=0x1a1d09648718 pc=0x562b725\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:245 +0x213 fp=0x1a1d096487e0 sp=0x1a1d09648748 pc=0x56269d3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096487e8 sp=0x1a1d096487e0 pc=0x5668501\n\ngoroutine 2 gp=0x1a1d09542d20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09648fa8 sp=0x1a1d09648f88 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x1a1d09648fe0 sp=0x1a1d09648fa8 pc=0x5626db3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09648fe8 sp=0x1a1d09648fe0 pc=0x5668501\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 18 gp=0x1a1d096843c0 m=nil [GC sweep wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644788 sp=0x1a1d09644768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:279 +0x94 fp=0x1a1d096447c8 sp=0x1a1d09644788 pc=0x560c694\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x1a1d096447e0 sp=0x1a1d096447c8 pc=0x55fd617\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096447e8 sp=0x1a1d096447e0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 19 gp=0x1a1d096845a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x1a1d09692000?, 0x1045f740?, 0x1?, 0x0?, 0x1a1d096845a0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644f78 sp=0x1a1d09644f58 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x17525960)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x1a1d09644fa8 sp=0x1a1d09644f78 pc=0x560a029\nruntime.bgscavenge(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:653 +0x3c fp=0x1a1d09644fc8 sp=0x1a1d09644fa8 pc=0x560a5bc\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x1a1d09644fe0 sp=0x1a1d09644fc8 pc=0x55fd5d7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09644fe8 sp=0x1a1d09644fe0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 3 gp=0x1a1d09543a40 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09649788 sp=0x1a1d09649768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x1a1d096497e0 sp=0x1a1d09649788 pc=0x5635487\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096497e8 sp=0x1a1d096497e0 pc=0x5668501\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\nrax 0x0\nrbx 0x7f52bbe48740\nrcx 0x7f52bbed694c\nrdx 0x6\nrdi 0xa70b\nrsi 0xa70b\nrbp 0xa70b\nrsp 0x7ffd4f7fae30\nr8 0x7ffd4f7faf00\nr9 0x7f52bbffc4e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7ffd4f7fb200\nr14 0x1a1d095421e0\nr15 0x10\nrip 0x7f52bbed694c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n7f52 bbf59cb1 ...nR.......R...\n00007ffd4f7fae70: 00007f52 6e7fc640 00007ffd 4f7fb140 @..nR...@..O....\n00007ffd4f7fae80: 00007ffd 4f7fafae 00007ffd 4f7fafaf ...O.......O....\n00007ffd4f7fae90: 00007f52 6e7fc640 00007f52 bbed47f5 @..nR....G..R...\n00007ffd4f7faea0: 00007f52 bc045c80 00000000 00000120 .\\..R... .......\n00007ffd4f7faeb0: 00000000 003d0f00 b406abf0 07e46d00 ..=......m......\n00007ffd4f7faec0: 00007f52 bbe48740 00000000 00000006 @...R...........\n00007ffd4f7faed0: 00000000 1a0a9f00 00007ffd 4f7fb200 ...........O....\n00007ffd4f7faee0: 00001a1d 095421e0 00007f52 bbe89646 .!T.....F...R...\n00007ffd4f7faef0: 00007f52 bc046e90 00007f52 bbe737f3 .n..R....7..R...\n00007ffd4f7faf00: 00000000 00000020 b406abf0 07e46d00 ........m......\n00007ffd4f7faf10: 00007f52 6dffc000 00007f52 bbf48ac4 ...mR.......R...\n00007ffd4f7faf20: 00000000 00000000 00000000 0000000d ................\n\ngoroutine 1 gp=0x1a1d095421e0 m=0 mp=0x17528da0 [running, locked to thread]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x1a1d096486a8 sp=0x1a1d09648698 pc=0x56668c8\nruntime.newm1(0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:2922 +0x85 fp=0x1a1d096486e8 sp=0x1a1d096486a8 pc=0x562b625\nruntime.newm(0x5635301?, 0x0, 0xd19a3c0?)\n\t/usr/lib/golang/src/runtime/proc.go:2902 +0x125 fp=0x1a1d09648718 sp=0x1a1d096486e8 pc=0x562b525\nruntime.startTemplateThread()\n\t/usr/lib/golang/src/runtime/proc.go:2947 +0x85 fp=0x1a1d09648748 sp=0x1a1d09648718 pc=0x562b725\nruntime.main()\n\t/usr/lib/golang/src/runtime/proc.go:245 +0x213 fp=0x1a1d096487e0 sp=0x1a1d09648748 pc=0x56269d3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096487e8 sp=0x1a1d096487e0 pc=0x5668501\n\ngoroutine 2 gp=0x1a1d09542d20 m=nil [force gc (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09648fa8 sp=0x1a1d09648f88 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.forcegchelper()\n\t/usr/lib/golang/src/runtime/proc.go:375 +0xb3 fp=0x1a1d09648fe0 sp=0x1a1d09648fa8 pc=0x5626db3\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09648fe8 sp=0x1a1d09648fe0 pc=0x5668501\ncreated by runtime.init.7 in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:363 +0x1a\n\ngoroutine 18 gp=0x1a1d096843c0 m=nil [GC sweep wait]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644788 sp=0x1a1d09644768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.bgsweep(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcsweep.go:279 +0x94 fp=0x1a1d096447c8 sp=0x1a1d09644788 pc=0x560c694\nruntime.gcenable.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x17 fp=0x1a1d096447e0 sp=0x1a1d096447c8 pc=0x55fd617\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096447e8 sp=0x1a1d096447e0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:214 +0x66\n\ngoroutine 19 gp=0x1a1d096845a0 m=nil [GC scavenge wait]:\nruntime.gopark(0x1a1d09692000?, 0x1045f740?, 0x1?, 0x0?, 0x1a1d096845a0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09644f78 sp=0x1a1d09644f58 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.(*scavengerState).park(0x17525960)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:425 +0x49 fp=0x1a1d09644fa8 sp=0x1a1d09644f78 pc=0x560a029\nruntime.bgscavenge(0x1a1d09692000)\n\t/usr/lib/golang/src/runtime/mgcscavenge.go:653 +0x3c fp=0x1a1d09644fc8 sp=0x1a1d09644fa8 pc=0x560a5bc\nruntime.gcenable.gowrap2()\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0x17 fp=0x1a1d09644fe0 sp=0x1a1d09644fc8 pc=0x55fd5d7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d09644fe8 sp=0x1a1d09644fe0 pc=0x5668501\ncreated by runtime.gcenable in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:215 +0xa5\n\ngoroutine 3 gp=0x1a1d09543a40 m=nil [GOMAXPROCS updater (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x1a1d09649788 sp=0x1a1d09649768 pc=0x566028e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:468\nruntime.updateMaxProcsGoroutine()\n\t/usr/lib/golang/src/runtime/proc.go:7095 +0xe7 fp=0x1a1d096497e0 sp=0x1a1d09649788 pc=0x5635487\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1a1d096497e8 sp=0x1a1d096497e0 pc=0x5668501\ncreated by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1\n\t/usr/lib/golang/src/runtime/proc.go:7083 +0x37\n\nrax 0x0\nrbx 0x7f52bbe48740\nrcx 0x7f52bbed694c\nrdx 0x6\nrdi 0xa70b\nrsi 0xa70b\nrbp 0xa70b\nrsp 0x7ffd4f7fae30\nr8 0x7ffd4f7faf00\nr9 0x7f52bbffc4e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7ffd4f7fb200\nr14 0x1a1d095421e0\nr15 0x10\nrip 0x7f52bbed694c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595527e8 sp=0x1077595527e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0x1077596043c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x10775961a740 sp=0x10775961a720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x10775961a7c8 sp=0x10775961a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961a7e0 sp=0x10775961a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961a7e8 sp=0x10775961a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0x1077597883c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559740 sp=0x107759559720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077595597c8 sp=0x107759559740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077595597e0 sp=0x1077595597c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595597e8 sp=0x1077595597e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0x1077597885a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559f40 sp=0x107759559f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759559fc8 sp=0x107759559f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759559fe0 sp=0x107759559fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759559fe8 sp=0x107759559fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0x1077599841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759552f40 sp=0x107759552f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759552fc8 sp=0x107759552f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759552fe0 sp=0x107759552fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759552fe8 sp=0x107759552fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0x1077596045a0 m=4 mp=0x107759582008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x10775961aea0 sp=0x10775961ae90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x10775961af40 sp=0x10775961aea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x10775961afc8 sp=0x10775961af40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961afe0 sp=0x10775961afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961afe8 sp=0x10775961afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 10 gp=0x107759788780 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207ccb99?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759616740 sp=0x107759616720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077596167c8 sp=0x107759616740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077596167e0 sp=0x1077596167c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077596167e8 sp=0x1077596167e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax 0x0\nrbx 0x7fd81f5da640\nrcx 0x7fd866a9c94c\nrdx 0x6\nrdi 0xa70c\nrsi 0xa715\nrbp 0xa715\nrsp 0x7fd81f5d9790\nr8 0x7fd81f5d9860\nr9 0x7fd866bc24e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7fd81f5d9b60\nr14 0x1077594510e0\nr15 0xffffffffffffffff\nrip 0x7fd866a9c94c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595527e8 sp=0x1077595527e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 34 gp=0x1077596043c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x10775961a740 sp=0x10775961a720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x10775961a7c8 sp=0x10775961a740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961a7e0 sp=0x10775961a7c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961a7e8 sp=0x10775961a7e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 8 gp=0x1077597883c0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559740 sp=0x107759559720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077595597c8 sp=0x107759559740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077595597e0 sp=0x1077595597c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077595597e8 sp=0x1077595597e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 9 gp=0x1077597885a0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759559f40 sp=0x107759559f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759559fc8 sp=0x107759559f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759559fe0 sp=0x107759559fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759559fe8 sp=0x107759559fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 19 gp=0x1077599841e0 m=nil [GC worker (idle)]:\nruntime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759552f40 sp=0x107759552f20 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x107759552fc8 sp=0x107759552f40 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x107759552fe0 sp=0x107759552fc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x107759552fe8 sp=0x107759552fe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 35 gp=0x1077596045a0 m=4 mp=0x107759582008 [GC mark termination]:\nruntime.systemstack_switch()\n\t/usr/lib/golang/src/runtime/asm_amd64.s:516 +0x8 fp=0x10775961aea0 sp=0x10775961ae90 pc=0x56668c8\nruntime.forEachP(...)\n\t/usr/lib/golang/src/runtime/proc.go:2112\nruntime.gcMarkDone()\n\t/usr/lib/golang/src/runtime/mgc.go:1042 +0x154 fp=0x10775961af40 sp=0x10775961aea0 pc=0x55fe474\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1912 +0x2c8 fp=0x10775961afc8 sp=0x10775961af40 pc=0x56002c8\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x10775961afe0 sp=0x10775961afc8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x10775961afe8 sp=0x10775961afe0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\ngoroutine 10 gp=0x107759788780 m=nil [GC worker (idle)]:\nruntime.gopark(0x7d9207ccb99?, 0x0?, 0x0?, 0x0?, 0x0?)\n\t/usr/lib/golang/src/runtime/proc.go:462 +0xce fp=0x107759616740 sp=0x107759616720 pc=0x566028e\nruntime.gcBgMarkWorker(0x1077595223f0)\n\t/usr/lib/golang/src/runtime/mgc.go:1791 +0xeb fp=0x1077596167c8 sp=0x107759616740 pc=0x56000eb\nruntime.gcBgMarkStartWorkers.gowrap1()\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x17 fp=0x1077596167e0 sp=0x1077596167c8 pc=0x55fffd7\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1771 +0x1 fp=0x1077596167e8 sp=0x1077596167e0 pc=0x5668501\ncreated by runtime.gcBgMarkStartWorkers in goroutine 1\n\t/usr/lib/golang/src/runtime/mgc.go:1695 +0x105\n\nrax 0x0\nrbx 0x7fd81f5da640\nrcx 0x7fd866a9c94c\nrdx 0x6\nrdi 0xa70c\nrsi 0xa715\nrbp 0xa715\nrsp 0x7fd81f5d9790\nr8 0x7fd81f5d9860\nr9 0x7fd866bc24e0\nr10 0x8\nr11 0x246\nr12 0x6\nr13 0x7fd81f5d9b60\nr14 0x1077594510e0\nr15 0xffffffffffffffff\nrip 0x7fd866a9c94c\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"041b58f88c07e6b6df5a78fe1b06fddf2a98976eca66b8f788c458fffb00b1ac","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"8d01f3d5bb1dffe043107fd710c91d72482c52b3696c1ab17cb3c16a039f0369","pr_number":7635,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"df0ed8cb7549a63439671f3dc385d910a4106ef5cb0378e7a7e6d06be9fbae28","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"97be445962a69349dfe2acc8f5e924b3c54887a318afb761923f76764db3b8d3","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.5641025641025643,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,1,1,1,0,2],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3ce657fb6660\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-f9dtl496nd4k\" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\",\n \"name\": \"5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:52:15.708697202Z\",\n \"endTime\": \"2026-10-23T14:11:23.696759282Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-f9dtl496nd4k--managed/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-f9dtl496nd4k--managed/providers/Microsoft.Network/networkInterfaces/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to http... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\n correlation ID: 2e16a443-f7c3-4087-8290-e874ac42ea22\n client request ID: d8c04265-7efa-4df2-aadc-80112544e6fc\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3ce657fb6660\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-f9dtl496nd4k\" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\",\n \"name\": \"5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:52:15.708697202Z\",\n \"endTime\": \"2026-10-23T14:11:23.696759282Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-f9dtl496nd4k--managed/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-f9dtl496nd4k--managed/providers/Microsoft.Network/networkInterfaces/i5l9u2w7t8e9k0t-np-1-jdwtb-bjlqw-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '5a34d2fd-f7db-460a-a272-334006a8a3b8' exceeded. Please try again after '2' seconds after additional tokens are available. Refer to http... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5eef3a72-7c1e-4bbd-a6fd-849f9e3ed01f\n correlation ID: 2e16a443-f7c3-4087-8290-e874ac42ea22\n client request ID: d8c04265-7efa-4df2-aadc-80112544e6fc\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x3fb2ef80a5e8\u003e: \n VerifyNodesReady failed: there are not ready nodes: cilium-cl-cilium-np-vlnxf-8twbs (status=False, reason=KubeletNotReady, message=container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, since=2026-10-23T11:44:16Z)\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-mj26w2hgsz8z\" to finish creating: GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\",\n \"name\": \"e205c3df-0cc8-46bb-84fb-519cf03956a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T11:25:56.40229713Z\",\n \"endTime\": \"2026-10-23T11:45:02.834813538Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-vlnxf-8twbs: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-vlnxf-8twbs: Failed: interfaces failed to create or update. err: failed to get existing resource cni-cilium-mj26w2hgsz8z--managed/cilium-cl-cilium-np-vlnxf-8twbs-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-mj26w2hgsz8z--managed/providers/Microsoft.Network/networkInterfaces/cilium-cl-cilium-np-vlnxf-8twbs-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://ak... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\n correlation ID: 98f92748-9012-4fff-87e0-a43e42d009b7\n client request ID: adb6cc59-3412-4269-9828-a7c6939df4dc\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0x3fb2ef80a5e8\u003e: \n VerifyNodesReady failed: there are not ready nodes: cilium-cl-cilium-np-vlnxf-8twbs (status=False, reason=KubeletNotReady, message=container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?, since=2026-10-23T11:44:16Z)\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-mj26w2hgsz8z\" to finish creating: GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\",\n \"name\": \"e205c3df-0cc8-46bb-84fb-519cf03956a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T11:25:56.40229713Z\",\n \"endTime\": \"2026-10-23T11:45:02.834813538Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-vlnxf-8twbs: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-vlnxf-8twbs: Failed: interfaces failed to create or update. err: failed to get existing resource cni-cilium-mj26w2hgsz8z--managed/cilium-cl-cilium-np-vlnxf-8twbs-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-mj26w2hgsz8z--managed/providers/Microsoft.Network/networkInterfaces/cilium-cl-cilium-np-vlnxf-8twbs-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://ak... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6849280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e205c3df-0cc8-46bb-84fb-519cf03956a7\n correlation ID: 98f92748-9012-4fff-87e0-a43e42d009b7\n client request ID: adb6cc59-3412-4269-9828-a7c6939df4dc\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113612033818628096","occurred_at":"2026-10-23T12:42:32Z","signature_id":"2105a054cb380d7c595411f3110d328751601f1b100d33affce72e45f10a0150","pr_number":7544,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113579232666849280","occurred_at":"2026-10-23T10:32:12Z","signature_id":"9813e169df513a9dded1ccb5b4bf7b1a057e15d6c3c68da4e0c1eb71f143dc1c","pr_number":6688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.5641025641025643,"anchor_occurrences":2,"after_last_push_count":21,"trend_counts":[0,0,5,8,0,3,2],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":54,"prior_last_seen_at":"2026-10-15T16:20:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-22-hbhgzqp5zds5/cluster-candidate-4-22-9t7f9b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2655fb8fc880\u003e: \n failed to create HCP cluster cluster-candidate-4-22-9t7f9b: failed waiting for cluster=\"cluster-candidate-4-22-9t7f9b\" in resourcegroup=\"rg-candidate-4-22-hbhgzqp5zds5\" to finish creating: GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\",\n \"name\": \"9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T19:29:36.620172443Z\",\n \"endTime\": \"2026-10-23T19:58:43.098929016Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\n correlation ID: 91833dfb-2346-485c-9abf-90635cc143aa\n client request ID: 8cd6038e-c1e2-4db7-aaf4-36204667b8f5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: HCP cluster rg-candidate-4-22-hbhgzqp5zds5/cluster-candidate-4-22-9t7f9b should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0x2655fb8fc880\u003e: \n failed to create HCP cluster cluster-candidate-4-22-9t7f9b: failed waiting for cluster=\"cluster-candidate-4-22-9t7f9b\" in resourcegroup=\"rg-candidate-4-22-hbhgzqp5zds5\" to finish creating: GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\",\n \"name\": \"9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T19:29:36.620172443Z\",\n \"endTime\": \"2026-10-23T19:58:43.098929016Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j5932672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9cab47e8-7ab4-4765-9ac8-c0235f97fa6d\n correlation ID: 91833dfb-2346-485c-9abf-90635cc143aa\n client request ID: 8cd6038e-c1e2-4db7-aaf4-36204667b8f5\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x29d3447c6480\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-gbjmnzzdfntv\" to finish creating: GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\",\n \"name\": \"644d90e9-61e3-42ef-a2dd-083bda41c8ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T17:53:58.718650875Z\",\n \"endTime\": \"2026-10-23T18:23:02.815422763Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\n correlation ID: 6d68ce4a-22a8-4eea-a977-b246a66b35ea\n client request ID: 9a628c19-f8fd-4bf6-859e-bb2f0567065e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:312]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0x29d3447c6480\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-gbjmnzzdfntv\" to finish creating: GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\",\n \"name\": \"644d90e9-61e3-42ef-a2dd-083bda41c8ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T17:53:58.718650875Z\",\n \"endTime\": \"2026-10-23T18:23:02.815422763Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8170368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/644d90e9-61e3-42ef-a2dd-083bda41c8ae\n correlation ID: 6d68ce4a-22a8-4eea-a977-b246a66b35ea\n client request ID: 9a628c19-f8fd-4bf6-859e-bb2f0567065e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"2871aba458073a19748dc486b5fa832486d5dacb815d2e076d0ba71820b99e15","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","signature_id":"7e722ad5a4ed9ab1a5b378b6eaaa935525e7b9867c93ed401690a68dd2ba6792","pr_number":7722,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.5641025641025643,"anchor_occurrences":2,"after_last_push_count":16,"trend_counts":[0,0,3,7,3,5,2],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":36,"prior_last_seen_at":"2026-10-16T09:12:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d7f216af3c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-sw22pzbqp5qg\" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\",\n \"name\": \"57d3456e-a011-478a-861a-14e4fe67f873\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:12:07.672503322Z\",\n \"endTime\": \"2026-10-23T09:31:10.853698527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\n correlation ID: 62c61aed-7617-418e-886e-be6374e7f22a\n client request ID: eb9543dc-eb4b-4b68-a493-4efa1c23921e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1d7f216af3c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-sw22pzbqp5qg\" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\",\n \"name\": \"57d3456e-a011-478a-861a-14e4fe67f873\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:12:07.672503322Z\",\n \"endTime\": \"2026-10-23T09:31:10.853698527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j9s2r8n0s7j5m0g-np-1-s7snv-swvmp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-sw22pzbqp5qg--managed/providers/Microsoft.Compute/virtualMachines/j9s2r8n0s7j5m0g-np-1-s7snv-swvmp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/57d3456e-a011-478a-861a-14e4fe67f873\n correlation ID: 62c61aed-7617-418e-886e-be6374e7f22a\n client request ID: eb9543dc-eb4b-4b68-a493-4efa1c23921e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-4m47cf\" for cluster \"cluster-hs-pre-4m47cf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x25f14c702960\u003e: \n failed to create NodePool np-4m47cf: failed waiting for nodepool=\"np-4m47cf\" for cluster \"cluster-hs-pre-4m47cf\" in resourcegroup=\"rg-hs-pre--phztwwdrkw7h\" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\",\n \"name\": \"d4f6c18f-891a-4c33-a2ba-8972525d31ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T07:15:22.43923977Z\",\n \"endTime\": \"2026-10-23T07:34:26.438466293Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\n correlation ID: dcef897a-e7d0-4e15-8934-33f203c26727\n client request ID: f4ae8e10-89f7-445a-9181-9c41d68ea128\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-4m47cf\" for cluster \"cluster-hs-pre-4m47cf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x25f14c702960\u003e: \n failed to create NodePool np-4m47cf: failed waiting for nodepool=\"np-4m47cf\" for cluster \"cluster-hs-pre-4m47cf\" in resourcegroup=\"rg-hs-pre--phztwwdrkw7h\" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\",\n \"name\": \"d4f6c18f-891a-4c33-a2ba-8972525d31ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T07:15:22.43923977Z\",\n \"endTime\": \"2026-10-23T07:34:26.438466293Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--phztwwdrkw7h-4m47cf--managed/providers/Microsoft.Compute/virtualMachines/s5d5j3u5l3o2v0o-np-4m47cf-8r2lc-bv7jq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/d4f6c18f-891a-4c33-a2ba-8972525d31ed\n correlation ID: dcef897a-e7d0-4e15-8934-33f203c26727\n client request ID: f4ae8e10-89f7-445a-9181-9c41d68ea128\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113545371434094592","occurred_at":"2026-10-23T08:17:38Z","signature_id":"ff19091ea6b6fd108dd7780cdd577078ceaa836eeffb4b21c27d5ef11633d976","pr_number":7687,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113514603643146240","occurred_at":"2026-10-23T06:15:23Z","signature_id":"72e291739086d9b63e2a0faa5afedbb7022d6a44908bff0d2b6ef6a7ca8a9ab5","pr_number":7709,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.5641025641025643,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-dzr6vp\"\nUnexpected error:\n \u003c*errors.joinError | 0x3fde2a0147b0\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x3fde2a4d4080\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x3fde2a4d4040\u003e{\n msg: \"failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x3fde2a4d4020\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x3fde2aa98140\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-dzr6vp\"\nUnexpected error:\n \u003c*errors.joinError | 0x3fde2a0147b0\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\": nodes \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x3fde2a4d4080\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x3fde2a4d4040\u003e{\n msg: \"failed to label node p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x3fde2a4d4020\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\": nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x3fde2aa98140\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"p2t3a2l0z2c2y2j-np-dzr6vp-49vrs-94cxm\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-l9g7mz\"\nUnexpected error:\n \u003c*errors.joinError | 0x2d180ff06060\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\": nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2d180f0a8800\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2d180f0a87a0\u003e{\n msg: \"failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2d180f0a8780\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x2d180ff5f680\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:250]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-l9g7mz\"\nUnexpected error:\n \u003c*errors.joinError | 0x2d180ff06060\u003e: \n VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\": nodes \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n {\n errs: [\n \u003c*fmt.wrapError | 0x2d180f0a8800\u003e{\n msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2d180f0a87a0\u003e{\n msg: \"failed to label node y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*fmt.wrapError | 0x2d180f0a8780\u003e{\n msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\": nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n err: \u003c*errors.StatusError | 0x2d180ff5f680\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"nodes \\\"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n Reason: \"Forbidden\",\n Details: {\n Name: \"y7i7w8o5j5g0n3s-np-l9g7mz-6xn8s-msvbg\",\n Group: \"\",\n Kind: \"nodes\",\n UID: \"\",\n Causes: nil,\n RetryAfterSeconds: 0,\n },\n Code: 403,\n },\n },\n },\n },\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7371/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113683564749918208","occurred_at":"2026-10-23T17:26:46Z","signature_id":"aa04e18b7fbd233f6ca4b1e47d82cf0953e640b1a2ab6e85b1475ab4208f342f","pr_number":7371,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113603186051780608","occurred_at":"2026-10-23T12:07:11Z","signature_id":"4cf06d6c9bb7b85aaa08e454a9e64270b9208afcbca137d23650a0ab18180e76","pr_number":7688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.5641025641025643,"anchor_occurrences":2,"after_last_push_count":17,"trend_counts":[0,0,5,8,0,2,2],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":60,"prior_last_seen_at":"2026-10-15T16:20:40Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":2}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T19:51:45Z\n Ended: 2026-10-23T20:15:45Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T19:51:45Z\n Ended: 2026-10-23T20:15:45Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5932672-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.248:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5932672/providers/microsoft.monitor/accounts/hcps-j5932672\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-hs2b5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tkmf8a67er4ddrbv9lckvi84f56n756-w1f1k6c5p9d8s2z/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T18:14:34Z\n Ended: 2026-10-23T18:39:29Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T18:14:35Z\n Ended: 2026-10-23T18:39:29Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8170368-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.219:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8170368/providers/microsoft.monitor/accounts/hcps-j8170368\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-d4dzl\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2tkl2ehrrq3bfgsofqiq3vb4ha9jrgif-l1v8c7t2c1o4j9z/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"bf583365df567c4528941bfba0cb382d9ef8ebcb3950c01907cf643a249725b2","pr_number":7712,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7722/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680070538170368","occurred_at":"2026-10-23T17:12:53Z","signature_id":"4b46ae0c9f8b5ffa392131441a033fd3d6fba4c8a02a7cf64403d6e16934d074","pr_number":7722,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.5641025641025643,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":17,"prior_last_seen_at":"2026-10-16T12:49:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1355449513d0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.000s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x1355449513d0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:58451-\u003e172.30.0.10:53: read: connection refused\",\n }","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.004s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x231711c56ee0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893-\u003e172.30.0.10:53: read: connection refused\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:391]: Timed out after 300.004s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0x231711c56ee0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893-\u003e172.30.0.10:53: read: connection refused\n {\n s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:48893-\u003e172.30.0.10:53: read: connection refused\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113735254979121152","occurred_at":"2026-10-23T20:52:10Z","signature_id":"24816c79684a0f74c02483d2e3b9365bf73a0c2a88310b47fae43e1866fb1d1a","pr_number":6770,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113545371434094592","occurred_at":"2026-10-23T08:17:38Z","signature_id":"3582f51ef43ab6cebf49738a8faa79e10138b0aa9aed91bb247ce640202ba0e7","pr_number":7687,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"95308b6c7143c4828d5cabfa013602f73aeedea67f64f8bfe3996fbde86741c3","failure_pattern":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","search_query":"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":2.5641025641025643,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,1,0,2,3,2],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-18T21:21:36Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","occurrences":2}],"full_error_samples":["time=2026-10-23T10:04:28.865Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-23T10:04:30.857Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-23T10:04:30.857Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-23T10:04:32.118Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-23T10:04:32.118Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4942976-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T10:04:32.118Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j4942976-svc\ntime=2026-10-23T10:19:36.095Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\"","time=2026-10-23T01:37:22.600Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n Kind: IstioUpgrade\\n AKSCluster: {{ svc.aks.name }}\\n\"\ntime=2026-10-23T01:37:23.985Z level=INFO msg=\"Istio upgrade -- cluster state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade k8sVersion=1.35 provisioningState=Succeeded installedRevisions=[asm-1-29] availableUpgrades=[asm-1-30] upgradeInProgress=false\ntime=2026-10-23T01:37:23.985Z level=INFO msg=\"Upgrading to svc.istio.versions target\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade from=asm-1-29 to=asm-1-30 k8sVersion=1.35\ntime=2026-10-23T01:37:24.555Z level=INFO msg=\"Istio upgrade -- mesh state\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade namespaces=0 controlPlanes=asm-1-29(2/2) ingressGateways=1 webhooks=1 configMaps=2\ntime=2026-10-23T01:37:24.555Z level=INFO msg=\"Starting canary -- installing target alongside current\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade cluster=ci01-j4358528-svc versions=asm-1-30 logger=istio-upgrade\ntime=2026-10-23T01:37:24.555Z level=INFO msg=\"Starting ARM operation\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade stepID=Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade operation=start-canary cluster=ci01-j4358528-svc\ntime=2026-10-23T01:52:27.816Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"failed to run istio upgrade step: failed to start canary: start-canary: ARM operation timed out or failed: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7711/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113559441314942976","occurred_at":"2026-10-23T09:13:33Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7711,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113434930414358528","occurred_at":"2026-10-23T00:58:47Z","signature_id":"00e0a2d839440451b084b8fbbf9281e66976dc3538ba8fb5256e47ec1f94c287","pr_number":7705,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-30T11:09:49Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: {}\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"8211b3315a6019168c62a8ef8d3269cdc9b89dc80385b17b9be39252d09088e9","pr_number":7463,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"8211b3315a6019168c62a8ef8d3269cdc9b89dc80385b17b9be39252d09088e9","pr_number":7463,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"e081efea8bc058d3864749e2214faab4cad1dba916314a282090b76fc0651463","failure_pattern":"Command failed after 5 attempts: oras cp registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config","search_query":"Command failed after 5 attempts: oras cp registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc --from-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json --to-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","occurrences":1}],"full_error_samples":["time=2026-10-23T04:51:07.246Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-10-23T04:52:17.695Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry registry.stage.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc to arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc.\\nThe image will still be available under it's original digest sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc in the target registry.\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError response from registry: recognizable error message not found: GET \\\"https://registry.stage.redhat.io/auth/realms/rhcc/protocol/redhat-docker-v2/auth?scope=repository%3Aoadp%2Foadp-velero-rhel9%3Apull\u0026service=docker-registry\\\": response status code 503: Service Unavailable\\nCommand failed after 5 attempts: oras cp registry.stage.redhat.io/oadp/oadp-velero-rhel9@sha256:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:3cd80bba1d9d323c025ef4ecf1412e348dddf0801af344e7839395073d16c7fc --from-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json --to-registry-config /tmp/tmp.SjGUwtkDHS/containers/auth.json\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113483940428779520","occurred_at":"2026-10-23T04:13:32Z","signature_id":"5351510afe6cf56e3a8b6c40bacbd2bff16093b6e23691ab45c119900b535d4b","pr_number":6770,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":38,"prior_last_seen_at":"2026-10-16T07:17:29Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-xhrpwn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1788046d3680\u003e: \n DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x178804888840\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:194]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-xhrpwn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1788046d3680\u003e: \n DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x178804888840\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-h2gpf.apps.aro.q1i8e4l6q2h3x2y.4cvg.j3270400.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"6541df12b028842ffaa548f5c83c0ed141122c488181c09a33e556c71ac2d3e8","pr_number":7463,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f7c740bf3534ca5945b9fa94518bc4b7a520a009eef4fbd7a1fc2602d0f2b958","failure_pattern":"DNS for route host agnhost-e2e-sample-app-\u003cid\u003e","search_query":"DNS for route host agnhost-e2e-sample-app-s57f2","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1f9142952260\u003e: \n DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1f9142952240\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: sample application must serve the exact customer-specified certificate with a valid Let's Encrypt chain\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x1f9142952260\u003e: \n DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n {\n msg: \"DNS for route host agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003c*fmt.wrapError | 0x1f9142952240\u003e{\n msg: \"DNS for agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-s57f2.apps.aro.ingress-cert-le.8n0g.j3146240.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113514603643146240","occurred_at":"2026-10-23T06:15:23Z","signature_id":"3e24e82e343cd7508ed60a7ec8730ea060e11e44c4fe22d5afc11ad0663e5eaa","pr_number":7709,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5e13e8ebe7676d236f83bd844b8c7a828a419da28309fcb2c22c74d629e1c635","failure_pattern":"Denial should come from the image registry policy, got: pod \"\u003cpod\u003e\" is forbidden: error looking up service account \u003cnamespace\u003e/default: serviceaccount \"default\" not found","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:113]: Denial should come from the image registry policy, got: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Image Registry Policy should deny pods with images from disallowed registries","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Image Registry Policy should deny pods with images from disallowed registries","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:113]: Denial should come from the image registry policy, got: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nExpected\n \u003cstring\u003e: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nto contain substring\n \u003cstring\u003e: not from an allowed registry\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:113]: Denial should come from the image registry policy, got: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nExpected\n \u003cstring\u003e: pods \"policy-test-disallowed\" is forbidden: error looking up service account image-policy-test-drtl2/default: serviceaccount \"default\" not found\nto contain substring\n \u003cstring\u003e: not from an allowed registry"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113601669802496000","occurred_at":"2026-10-23T12:01:21Z","signature_id":"f653863f476c685d13e3a7415f0b0b11a8654a0f134f21f96f43a33cb5375be8","pr_number":7611,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"11f1e4870cb02ff7036064941db2a3263ef1df1d64949078bceae777a2c26ab6","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-b... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cda231b9b40\u003e: \n failed to create NodePool np-d4pgmj: failed waiting for nodepool=\"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\" in resourcegroup=\"rg-candidate-4-22-jrp5tdgcl9kv\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n \"name\": \"f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:58:53.825112936Z\",\n \"endTime\": \"2026-10-23T13:17:55.550011725Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7q7k8t3p4d7j9m-np-d4pgmj-4fc64-lwc4j: Failed: availabilitysets failed to create or update. err: failed to get existing resource rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/providers/Microsoft.Compute/availabilitySets/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-b... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n correlation ID: bb68fb60-f553-4611-a472-1dc9f8329c89\n client request ID: 638d8201-2360-4c79-a5aa-4979c9f07d33\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3cda231b9b40\u003e: \n failed to create NodePool np-d4pgmj: failed waiting for nodepool=\"np-d4pgmj\" for cluster \"cluster-candidate-4-22-d4pgmj\" in resourcegroup=\"rg-candidate-4-22-jrp5tdgcl9kv\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n \"name\": \"f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:58:53.825112936Z\",\n \"endTime\": \"2026-10-23T13:17:55.550011725Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7q7k8t3p4d7j9m-np-d4pgmj-4fc64-lwc4j: Failed: availabilitysets failed to create or update. err: failed to get existing resource rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jrp5tdgcl9kv-candidate-d4pgmj--managed/providers/Microsoft.Compute/availabilitySets/2tkgip06nnkc4a8745pcppbif806qoqb_y7q7k8t3p4d7j9m-np-d4pgmj-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-b... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f456ad88-f3c9-4971-9d6b-4fae5a05e8b0\n correlation ID: bb68fb60-f553-4611-a472-1dc9f8329c89\n client request ID: 638d8201-2360-4c79-a5aa-4979c9f07d33\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113599326222553088","occurred_at":"2026-10-23T11:52:02Z","signature_id":"4b66edeccfb70c1f5cfd0f1d6ba0073696d1cd6c7003d445193ae337daf93d50","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"146ba2f03b8e73d8b9aa488d4c220ca3b04255c2372536bbb59e6aa3a97d9e10","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Ple... message truncated Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b310ced2400\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-d2zgjcg49b99\" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\",\n \"name\": \"8b884d06-af05-4a79-8c59-961aac29861b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T07:12:27.383685402Z\",\n \"endTime\": \"2026-10-23T07:31:36.955210198Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-vxg9v: Failed: availabilitysets failed to create or update. err: failed to get existing resource clusternp128-d2zgjcg49b99--managed/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/availabilitySets/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Ple... message truncated\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n correlation ID: 07968fa2-a7d8-4604-9db4-9bbf3995867c\n client request ID: f057ae85-7143-412c-b3d7-6a49973e5447\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b310ced2400\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-d2zgjcg49b99\" to finish creating: GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\",\n \"name\": \"8b884d06-af05-4a79-8c59-961aac29861b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T07:12:27.383685402Z\",\n \"endTime\": \"2026-10-23T07:31:36.955210198Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-vxg9v: Failed: availabilitysets failed to create or update. err: failed to get existing resource clusternp128-d2zgjcg49b99--managed/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/availabilitySets/2tkbh29lva3024obtdev5eh0n4ambooh_k6m4e0t9w1v5g7p-nodepool-128gib-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'db27175c-5bd0-48b4-929a-41de9a53ffbf' exceeded. Ple... message truncated\\nMachine k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-d2zgjcg49b99--managed/providers/Microsoft.Compute/virtualMachines/k6m4e0t9w1v5g7p-nodepool-128gib-qjbwx-8pg8v to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3146240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8b884d06-af05-4a79-8c59-961aac29861b\n correlation ID: 07968fa2-a7d8-4604-9db4-9bbf3995867c\n client request ID: f057ae85-7143-412c-b3d7-6a49973e5447\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113514603643146240","occurred_at":"2026-10-23T06:15:23Z","signature_id":"cbafe1b07c5b593f659af5b88993e3571384544466e1095e0be4c9ecc5336c65","pr_number":7709,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"bf2604f0a57cc83f42180ea2d978fdb012e4de2c6d11e90a5b15c16b2e73342d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '1' sec... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x236cf2ed0920\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-46ntd4r5fjxr\" to finish creating: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\",\n \"name\": \"993068cc-46a0-4e4b-9083-bf129b4ea866\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:05:04.420745783Z\",\n \"endTime\": \"2026-10-23T12:24:12.746383509Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-cm2dh-2kqqm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-cm2dh-2kqqm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine private-ingress-np-1-cm2dh-2kqqm: Failed: availabilitysets failed to create or update. err: failed to get existing resource private-ingress-46ntd4r5fjxr--managed/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/availabilitySets/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '1' sec... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n correlation ID: d70fefde-ba5d-4298-ad39-0d96bb22c447\n client request ID: 9ac09fab-3463-40c0-9c76-35eeb3b20d7a\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x236cf2ed0920\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-46ntd4r5fjxr\" to finish creating: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\",\n \"name\": \"993068cc-46a0-4e4b-9083-bf129b4ea866\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:05:04.420745783Z\",\n \"endTime\": \"2026-10-23T12:24:12.746383509Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-cm2dh-2kqqm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-cm2dh-2kqqm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine private-ingress-np-1-cm2dh-2kqqm: Failed: availabilitysets failed to create or update. err: failed to get existing resource private-ingress-46ntd4r5fjxr--managed/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-46ntd4r5fjxr--managed/providers/Microsoft.Compute/availabilitySets/2tkfr2hi8cllpnqtjifbtjedfcdp3pun_private-ingress-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b26c2343-0863-4a55-bb7e-dfb2544999c1' exceeded. Please try again after '1' sec... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/993068cc-46a0-4e4b-9083-bf129b4ea866\n correlation ID: d70fefde-ba5d-4298-ad39-0d96bb22c447\n client request ID: 9ac09fab-3463-40c0-9c76-35eeb3b20d7a\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"d010fa9f43f9777168167e02e99bc932a1d4357cede95310e395869051417e5a","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7d25cf9b217ca7a1560a0ac8f986a95cd4726bd1277cdf1dc11174901dd19f0f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '10' se... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1909134f0c20\u003e: \n failed to create NodePool np-qptx5c: failed waiting for nodepool=\"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\" in resourcegroup=\"rg-candidate-5-1-6mzh9r6brjfl\" to finish creating: GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n \"name\": \"1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:34:04.539419115Z\",\n \"endTime\": \"2026-10-23T13:53:12.193747284Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Compute/virtualMachines/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Network/networkInterfaces/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' se... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n correlation ID: 65b99ffd-0c17-417b-bd92-7368727e5721\n client request ID: 5a9e3ea8-7ac7-474e-85b1-9320f2c4be29\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1909134f0c20\u003e: \n failed to create NodePool np-qptx5c: failed waiting for nodepool=\"np-qptx5c\" for cluster \"cluster-candidate-5-1-qptx5c\" in resourcegroup=\"rg-candidate-5-1-6mzh9r6brjfl\" to finish creating: GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n \"name\": \"1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:34:04.539419115Z\",\n \"endTime\": \"2026-10-23T13:53:12.193747284Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Compute/virtualMachines/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-6mzh9r6brjfl-candidate-qptx5c--managed/providers/Microsoft.Network/networkInterfaces/q5r0t5o1y0m4h2g-np-qptx5c-kw7s7-czc5k-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '0d25d885-2cd8-4972-b610-4d85881c1ec4' exceeded. Please try again after '10' se... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9339904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1b3eb82d-60f1-4e1d-93f9-e0f48de0d014\n correlation ID: 65b99ffd-0c17-417b-bd92-7368727e5721\n client request ID: 5a9e3ea8-7ac7-474e-85b1-9320f2c4be29\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6718/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113594282169339904","occurred_at":"2026-10-23T11:32:00Z","signature_id":"c11a2050e197d4a44a7c09a12f94ca83ff56f060fe76b2742e4b18d0997beadb","pr_number":6718,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"802121af546da5e12fc3cac43502a22a348ae1f3e72897092323375f66a04dbc","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after addi... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x99f918edcc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vknb2mpvr5xz\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\",\n \"name\": \"76509bdd-86ae-4734-aa51-d853651ea0fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:59:15.455605547Z\",\n \"endTime\": \"2026-10-23T13:18:15.598008978Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-hc6wj-5qqzr: Failed: availabilitysets failed to create or update. err: failed to get existing resource v20260901preview-vknb2mpvr5xz--managed/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-vknb2mpvr5xz--managed/providers/Microsoft.Compute/availabilitySets/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after addi... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n correlation ID: 82980038-7d5f-4ddb-bc41-a8d67ca04b71\n client request ID: 6668afc8-6cc9-4e23-9e4f-4e194ede10a5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x99f918edcc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vknb2mpvr5xz\" to finish creating: GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\",\n \"name\": \"76509bdd-86ae-4734-aa51-d853651ea0fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:59:15.455605547Z\",\n \"endTime\": \"2026-10-23T13:18:15.598008978Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-hc6wj-5qqzr: Failed: availabilitysets failed to create or update. err: failed to get existing resource v20260901preview-vknb2mpvr5xz--managed/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-vknb2mpvr5xz--managed/providers/Microsoft.Compute/availabilitySets/2tkgjffnjg9mp3of67usueutl55rqvo7_v20260901-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '10' seconds after addi... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2553088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/76509bdd-86ae-4734-aa51-d853651ea0fd\n correlation ID: 82980038-7d5f-4ddb-bc41-a8d67ca04b71\n client request ID: 6668afc8-6cc9-4e23-9e4f-4e194ede10a5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113599326222553088","occurred_at":"2026-10-23T11:52:02Z","signature_id":"b2e5ac734561cd50d6d2bb64f76bb0716f87849cddae08ac51680aa3c5151ea8","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e9ecd96fd76a659f151ac5df30d476a4d8045c25ee424b28225788c72e8971d2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refe... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x37701b1c12c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-snrv785b6qc8\" to finish creating: GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n \"name\": \"dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:27:43.519923521Z\",\n \"endTime\": \"2026-10-23T09:46:52.566133312Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-snrv785b6qc8--managed/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-snrv785b6qc8--managed/providers/Microsoft.Network/networkInterfaces/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refe... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n correlation ID: f689e4cc-97e8-4962-aa14-27d872557ae0\n client request ID: d83db591-fd3e-4280-8e24-4c7c60560e28\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x37701b1c12c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-snrv785b6qc8\" to finish creating: GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n \"name\": \"dc59c574-bb7b-4b97-8a05-f66d8dd763d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:27:43.519923521Z\",\n \"endTime\": \"2026-10-23T09:46:52.566133312Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-snrv785b6qc8--managed/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-snrv785b6qc8--managed/providers/Microsoft.Network/networkInterfaces/r4y1c0t2q5b2l1a-np-1-dfjw9-stwlm-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are available. Refe... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j1801600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/dc59c574-bb7b-4b97-8a05-f66d8dd763d0\n correlation ID: f689e4cc-97e8-4962-aa14-27d872557ae0\n client request ID: d83db591-fd3e-4280-8e24-4c7c60560e28\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113546001531801600","occurred_at":"2026-10-23T08:20:08Z","signature_id":"40c4710e56e4b7283917fa74062c14639587a07ba8e4e6a92442bf16cdee182b","pr_number":7709,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6a41a55832324284d10413d9a99da8343f3b8c1a9dc53d5ecf869bc08ac34982","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer t... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x19a5a6aee900\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-662w726xqd2k\" to finish creating: GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\",\n \"name\": \"91414204-0657-405d-a101-7608067a7216\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T15:54:03.184233444Z\",\n \"endTime\": \"2026-10-23T16:13:03.632601143Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Compute/virtualMachines/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-662w726xqd2k--managed/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Network/networkInterfaces/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '1' seconds after additional tokens are available. Refer t... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n correlation ID: 47f6ba53-01c6-4790-943f-535cc7879659\n client request ID: 259bc435-7233-4d58-9c0f-3d51379cd131\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x19a5a6aee900\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-662w726xqd2k\" to finish creating: GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\",\n \"name\": \"91414204-0657-405d-a101-7608067a7216\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T15:54:03.184233444Z\",\n \"endTime\": \"2026-10-23T16:13:03.632601143Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Compute/virtualMachines/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o1v4y4o7u7o3r7p-np-1-jnmqg-x488v: Failed: interfaces failed to create or update. err: failed to get existing resource pullsecret-test-662w726xqd2k--managed/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-662w726xqd2k--managed/providers/Microsoft.Network/networkInterfaces/o1v4y4o7u7o3r7p-np-1-jnmqg-x488v-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '39eefc4f-5e24-491d-9f09-bcc0ad573bcf' exceeded. Please try again after '1' seconds after additional tokens are available. Refer t... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2459904.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91414204-0657-405d-a101-7608067a7216\n correlation ID: 47f6ba53-01c6-4790-943f-535cc7879659\n client request ID: 259bc435-7233-4d58-9c0f-3d51379cd131\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7656/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113649065022459904","occurred_at":"2026-10-23T15:09:40Z","signature_id":"1ddc8754dde4d08e908579fef94b5559366fe2b19d0cf98ab8ba070611484821","pr_number":7656,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"5bd4868e5b799ebb3599c1c4a1a9e5adf369cd18ab421a17e5fc658ab1e93070","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. Refer... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x174cbf30c80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-nwv4f2dxcv4b\" to finish creating: GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\",\n \"name\": \"7be20667-78b5-4033-a3f1-3a092d6628f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:40:10.449361114Z\",\n \"endTime\": \"2026-10-23T09:59:13.321219203Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-nwv4f2dxcv4b--managed/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-nwv4f2dxcv4b--managed/providers/Microsoft.Network/networkInterfaces/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available. Refer... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n correlation ID: 6164f95f-1b20-49d8-a767-33fc06ba3b2f\n client request ID: 11eb729f-0d89-4edd-a41e-8cb113738ef8\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x174cbf30c80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-nwv4f2dxcv4b\" to finish creating: GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\",\n \"name\": \"7be20667-78b5-4033-a3f1-3a092d6628f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:40:10.449361114Z\",\n \"endTime\": \"2026-10-23T09:59:13.321219203Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz: Failed: interfaces failed to create or update. err: failed to get existing resource private-keyvault-nwv4f2dxcv4b--managed/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-nwv4f2dxcv4b--managed/providers/Microsoft.Network/networkInterfaces/v2y3z7y3d3l2e0r-np-1-4j7f7-brrbz-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'f8be0ede-39df-41cf-aed7-7f3626f23a5a' exceeded. Please try again after '3' seconds after additional tokens are available. Refer... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9447296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be20667-78b5-4033-a3f1-3a092d6628f5\n correlation ID: 6164f95f-1b20-49d8-a767-33fc06ba3b2f\n client request ID: 11eb729f-0d89-4edd-a41e-8cb113738ef8\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113548717339447296","occurred_at":"2026-10-23T08:30:48Z","signature_id":"eac87251b067df59431cf544f3a76dfd222fded34dfba939a62a417723a28785","pr_number":7665,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0183c80982ac91e40ea21123d1f1a20353ca5fecf0db33e487b670cf6d4d4996","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x24b9ff37f340\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xqmhq7fznl4t\" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n \"name\": \"90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T00:52:55.985747163Z\",\n \"endTime\": \"2026-10-23T01:11:56.482302149Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-brczf to exist\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-z45q7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-z45q7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xqmhq7fznl4t--managed/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Network/networkInterfaces/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n correlation ID: 980d0943-0968-489f-acea-10a235919d44\n client request ID: 9fc719e7-58e9-4f80-97ec-7d7310001822\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x24b9ff37f340\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xqmhq7fznl4t\" to finish creating: GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n \"name\": \"90f18bf3-8ce0-485d-ade8-a5cce140bcbb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T00:52:55.985747163Z\",\n \"endTime\": \"2026-10-23T01:11:56.482302149Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-brczf to exist\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-z45q7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Compute/virtualMachines/q5x5a0r9a3c8o8m-np-1-qct67-z45q7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5x5a0r9a3c8o8m-np-1-qct67-brczf: Failed: interfaces failed to create or update. err: failed to get existing resource autoscaling-cluster-xqmhq7fznl4t--managed/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-xqmhq7fznl4t--managed/providers/Microsoft.Network/networkInterfaces/q5x5a0r9a3c8o8m-np-1-qct67-brczf-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '10' seconds after additional tokens are available... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j7006464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/90f18bf3-8ce0-485d-ade8-a5cce140bcbb\n correlation ID: 980d0943-0968-489f-acea-10a235919d44\n client request ID: 9fc719e7-58e9-4f80-97ec-7d7310001822\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113420718787006464","occurred_at":"2026-10-23T00:01:14Z","signature_id":"0d9828b0bb1f27d9cc3639f58f494971e86d707f6ec7be8d51e1bfc562947fba","pr_number":7688,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"8587d6ef8cd7dac5fe4005b92f7045c941c29895bdc330fc36d748cf48c16a17","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xa64ef1027e0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-h54k9n8pjkjm\" to finish creating: GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\",\n \"name\": \"1e0c8c9f-614e-479d-8628-81206456e882\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T17:54:15.481914664Z\",\n \"endTime\": \"2026-10-23T18:23:16.346423782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n correlation ID: 11ba9467-ece2-482a-942e-01e77f78ac77\n client request ID: d489ddc3-32d0-4662-9c28-60d44153b44d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xa64ef1027e0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-h54k9n8pjkjm\" to finish creating: GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\",\n \"name\": \"1e0c8c9f-614e-479d-8628-81206456e882\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T17:54:15.481914664Z\",\n \"endTime\": \"2026-10-23T18:23:16.346423782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9103104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/1e0c8c9f-614e-479d-8628-81206456e882\n correlation ID: 11ba9467-ece2-482a-942e-01e77f78ac77\n client request ID: d489ddc3-32d0-4662-9c28-60d44153b44d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","signature_id":"3cd718d00c36786db9c07977d8d9bab6da7e3aa088465ca79459dcdaa748a25b","pr_number":7547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":11,"trend_counts":[0,0,0,1,1,4,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":23,"prior_last_seen_at":"2026-10-14T19:09:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x2987b030ca40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rzw7qwnn5qjc\" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n \"name\": \"76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T03:22:03.46163902Z\",\n \"endTime\": \"2026-10-23T03:51:08.692840996Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 23m19s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n correlation ID: bb01406a-f12b-4a31-a989-90ce9d573bdf\n client request ID: f0973675-7d1b-4ddd-b467-290c8a664a39\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0x2987b030ca40\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-rzw7qwnn5qjc\" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n \"name\": \"76e4548a-7fa4-49e0-977c-056006f3a3ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T03:22:03.46163902Z\",\n \"endTime\": \"2026-10-23T03:51:08.692840996Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 23m19s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e4548a-7fa4-49e0-977c-056006f3a3ed\n correlation ID: bb01406a-f12b-4a31-a989-90ce9d573bdf\n client request ID: f0973675-7d1b-4ddd-b467-290c8a664a39\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113459428752625664","occurred_at":"2026-10-23T02:36:08Z","signature_id":"a0d813b5252e54924e130bf68e119ec22802307c2b8775c3330243b5fac10ac1","pr_number":7687,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-28","2026-10-12"],"prior_runs_affected":7,"prior_last_seen_at":"2026-10-14T12:43:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x10298a136780\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-thv2btpwv2x8\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\",\n \"name\": \"505bd364-3947-46d9-aabe-9bb9ef0e8d13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:20:41.038643697Z\",\n \"endTime\": \"2026-10-23T18:49:43.826799473Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 9m9s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\n correlation ID: fb84c414-e895-47e3-bdef-c718ca7da71b\n client request ID: e0f67746-2722-4f93-b4a1-3c122c4d2e69\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:94]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x10298a136780\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-thv2btpwv2x8\" to finish creating: GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\",\n \"name\": \"505bd364-3947-46d9-aabe-9bb9ef0e8d13\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T18:20:41.038643697Z\",\n \"endTime\": \"2026-10-23T18:49:43.826799473Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 9m9s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j9317632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/505bd364-3947-46d9-aabe-9bb9ef0e8d13\n correlation ID: fb84c414-e895-47e3-bdef-c718ca7da71b\n client request ID: e0f67746-2722-4f93-b4a1-3c122c4d2e69\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7657/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113680800309317632","occurred_at":"2026-10-23T17:15:47Z","signature_id":"e1da39dbfa2686ab72adc9980862457714384b0d27434b03e987e157712cc1b9","pr_number":7657,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1cd304b95c756f8a1f8ad9e9b7d8205b8dcc2ba19e257386e89c7f15d0fddc01","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":10,"prior_last_seen_at":"2026-10-16T05:08:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x35e0b0894420\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-9pzwhwthgd8h\" to finish creating: GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\",\n \"name\": \"4735d84e-eeda-4991-9a00-15bdfb0112df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T22:59:37.48096278Z\",\n \"endTime\": \"2026-10-23T23:18:39.961096304Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n correlation ID: e810e9b2-3fca-4c24-b421-0b45eb552e0e\n client request ID: 0af33e2e-2abb-488f-a355-27d28c97d1c5\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0x35e0b0894420\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-9pzwhwthgd8h\" to finish creating: GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\",\n \"name\": \"4735d84e-eeda-4991-9a00-15bdfb0112df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T22:59:37.48096278Z\",\n \"endTime\": \"2026-10-23T23:18:39.961096304Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-s9cv2 to exist\\nMachine r0o3j9w3o6f8u8z-np-1-x89sx-xbh47: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-9pzwhwthgd8h--managed/providers/Microsoft.Compute/virtualMachines/r0o3j9w3o6f8u8z-np-1-x89sx-xbh47 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2135936.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4735d84e-eeda-4991-9a00-15bdfb0112df\n correlation ID: e810e9b2-3fca-4c24-b421-0b45eb552e0e\n client request ID: 0af33e2e-2abb-488f-a355-27d28c97d1c5\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113746348372135936","occurred_at":"2026-10-23T21:36:15Z","signature_id":"0df12b245e7e51211310f93e12a460611030c5368b7e9d6d2e212e20fb97294f","pr_number":7733,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"2913b68eb188eee155973da8f2278b815824c08e26b8568b029df43566c7bce9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xfba86cf63e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-gkpx2dhvd6bm\" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\",\n \"name\": \"7f0ee636-256c-422d-a7d4-36addd512bcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:14:45.906565692Z\",\n \"endTime\": \"2026-10-23T09:33:49.808866171Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-m7jn2-ljktd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-m7jn2-ljktd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n correlation ID: 02fd815c-7a91-4e34-a2c7-a3013c24d175\n client request ID: f1495231-4c88-4b44-9a23-53972ede44a2\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xfba86cf63e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-gkpx2dhvd6bm\" to finish creating: GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\",\n \"name\": \"7f0ee636-256c-422d-a7d4-36addd512bcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:14:45.906565692Z\",\n \"endTime\": \"2026-10-23T09:33:49.808866171Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-m7jn2-ljktd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-m7jn2-ljktd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-gkpx2dhvd6bm--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-m7jn2-ljktd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j4094592.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7f0ee636-256c-422d-a7d4-36addd512bcd\n correlation ID: 02fd815c-7a91-4e34-a2c7-a3013c24d175\n client request ID: f1495231-4c88-4b44-9a23-53972ede44a2\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113545371434094592","occurred_at":"2026-10-23T08:17:38Z","signature_id":"f9315d1a908c869487b8528009d6e3be71e485ce7d3c349af0d4312ee843c244","pr_number":7687,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f957ba9aa3421e976868190e9cbb9bb30db37080a11af2efd3d47ffe3cc63196","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: NotReady: * NodeHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,2,1,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":6,"prior_last_seen_at":"2026-10-16T20:03:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x159fda57b1e0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-972dcv\" in resourcegroup=\"rg-np-version-upgrade-972dcv-p4hg8tz7gs76\" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\",\n \"name\": \"d066a71c-277e-4671-a4dd-16329cfd2682\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T14:16:35.253267721Z\",\n \"endTime\": \"2026-10-23T14:35:40.055586863Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n correlation ID: df6d449d-b4ac-498f-8d50-5d981e7f434b\n client request ID: 09867594-8f78-4da8-bf75-c401ddee2f7d\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.42\nUnexpected error:\n \u003c*fmt.wrapError | 0x159fda57b1e0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-972dcv\" in resourcegroup=\"rg-np-version-upgrade-972dcv-p4hg8tz7gs76\" to finish creating: GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\",\n \"name\": \"d066a71c-277e-4671-a4dd-16329cfd2682\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T14:16:35.253267721Z\",\n \"endTime\": \"2026-10-23T14:35:40.055586863Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x1s8z7k3u9m0n1x-npupgrade-4-20-j62c2-jk84f: NotReady: * NodeHealthy:\\n * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j8628096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d066a71c-277e-4671-a4dd-16329cfd2682\n correlation ID: df6d449d-b4ac-498f-8d50-5d981e7f434b\n client request ID: 09867594-8f78-4da8-bf75-c401ddee2f7d\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7544/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113612033818628096","occurred_at":"2026-10-23T12:42:32Z","signature_id":"d9cde6d3be6b80b67637a1674fb67e65e7c5c386976a816549974fb5fd2ebc69","pr_number":7544,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f3a0ec083a80f290f7c051e6f9dfe7391746cc2a0f62d0ca2697a0285007d4a5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-03T18:55:53Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a2ce0502aa0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2q9b4whtt4bx\" to finish creating: GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n \"name\": \"ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:46:54.928693784Z\",\n \"endTime\": \"2026-10-23T10:05:56.71052837Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: Creating: virtualmachine creating or updating\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n correlation ID: b9eeb7ac-2571-4ebc-8877-daaa0fdd2739\n client request ID: 12c440df-92bf-4a8e-95cd-5978adfb18ec\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x3a2ce0502aa0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2q9b4whtt4bx\" to finish creating: GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n \"name\": \"ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T09:46:54.928693784Z\",\n \"endTime\": \"2026-10-23T10:05:56.71052837Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 3 machines are not healthy\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 3 machines are not ready\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-h6wtc: Creating: virtualmachine creating or updating\\nMachine j3b3f4v2z3z1f7z-autoscale-noaz-sk4rk-p8ssn: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j3739264.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ac453c5e-6fb7-49c6-8bb6-3c58ccee2dd6\n correlation ID: b9eeb7ac-2571-4ebc-8877-daaa0fdd2739\n client request ID: 12c440df-92bf-4a8e-95cd-5978adfb18ec\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7708/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113549009753739264","occurred_at":"2026-10-23T08:32:06Z","signature_id":"504e0ef2def37e129dc9af51f3cea8c5643dc0df0434eba4232abc7a90d05a61","pr_number":7708,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,1,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-12"],"prior_runs_affected":3,"prior_last_seen_at":"2026-10-16T09:20:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x12108ffe4840\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gmjhx5ggwwt9\" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\",\n \"name\": \"74fee9f7-b402-4d5d-b683-6e4318e06904\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T03:34:18.354098626Z\",\n \"endTime\": \"2026-10-23T03:53:18.691661116Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\n correlation ID: b1f52151-27ba-447c-87a0-0859e9b312e7\n client request ID: f918cf71-16c2-4813-9b6f-6f13fae950d6\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x12108ffe4840\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gmjhx5ggwwt9\" to finish creating: GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\",\n \"name\": \"74fee9f7-b402-4d5d-b683-6e4318e06904\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T03:34:18.354098626Z\",\n \"endTime\": \"2026-10-23T03:53:18.691661116Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gmjhx5ggwwt9--managed/providers/Microsoft.Compute/virtualMachines/g0n4d9o1a7i5k4z-autoscale-az-x494g-g7h6z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j2625664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74fee9f7-b402-4d5d-b683-6e4318e06904\n correlation ID: b1f52151-27ba-447c-87a0-0859e9b312e7\n client request ID: f918cf71-16c2-4813-9b6f-6f13fae950d6\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7687/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113459428752625664","occurred_at":"2026-10-23T02:36:08Z","signature_id":"361d3eae8393f6eef155ba76cade65d4102abb09f4a193a37d5f4161c2bf94bf","pr_number":7687,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b0003cc8ca3a9f7347553bcc0638ed02c32204aa3c9fd097c1f10d2bf5392fbd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc108f3a63c0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-9rjzw7\" in resourcegroup=\"private-ingress-zqfzzkb8lcwr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20261023T175235Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9bc1794b-fa34-4d7e-af9a-8ae43e2a8056', request correlation id '12c52ee3-c818-4a7a-b25e-ae843a2763d9'.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n correlation ID: f4e0dcee-95e8-42c0-8753-7683f8dd661f\n client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc108f3a63c0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-9rjzw7\" in resourcegroup=\"private-ingress-zqfzzkb8lcwr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"InternalServerError\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20261023T175235Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9bc1794b-fa34-4d7e-af9a-8ae43e2a8056', request correlation id '12c52ee3-c818-4a7a-b25e-ae843a2763d9'.\\\"\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-zqfzzkb8lcwr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-9rjzw7/operationStatuses/08584088289957855365\n correlation ID: f4e0dcee-95e8-42c0-8753-7683f8dd661f\n client request ID: unavailable\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","signature_id":"86725f75fec081c22e7a197ddb0df251563b2a69bc88d75ade0edf6269655c72","pr_number":7547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,3,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":18,"prior_last_seen_at":"2026-10-13T15:35:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x3a796e6c5e8\u003e: \n resource group \"v20260901preview-7822pl2jslgp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-7822pl2jslgp\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\",\n \"name\": \"aa1c3345-b774-4a31-837e-23361b1b3df1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:19:48.083110134Z\",\n \"endTime\": \"2026-10-23T12:43:54.717460671Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkfr4jhp6k8h2l5ht0m1oco8hjk189n still exists (deletion dispatched at 2026-10-23T12:19:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n correlation ID: d70e853f-388e-4661-a9cc-9552b311d878\n client request ID: 94a8bbf5-a064-4508-a581-559607551023\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x3a796e6c5e8\u003e: \n resource group \"v20260901preview-7822pl2jslgp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-7822pl2jslgp\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\",\n \"name\": \"aa1c3345-b774-4a31-837e-23361b1b3df1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T12:19:48.083110134Z\",\n \"endTime\": \"2026-10-23T12:43:54.717460671Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkfr4jhp6k8h2l5ht0m1oco8hjk189n still exists (deletion dispatched at 2026-10-23T12:19:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/aa1c3345-b774-4a31-837e-23361b1b3df1\n correlation ID: d70e853f-388e-4661-a9cc-9552b311d878\n client request ID: 94a8bbf5-a064-4508-a581-559607551023\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"2623387444ac60aad1c81d411318d579ab144dfc53c580add9b2105a4d1cf697","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":18,"trend_counts":[0,0,1,5,3,2,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":138,"prior_last_seen_at":"2026-10-13T15:48:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x36fd45eaa060\u003e: \n resource group \"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-jdxf2b\" in resourcegroup=\"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\",\n \"name\": \"9861ff20-e277-4831-93f1-1162afbd108a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:01:17.842344425Z\",\n \"endTime\": \"2026-10-23T13:25:21.427910502Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkg8fusv1drnq89nr4jr4kc118h003a still exists (deletion dispatched at 2026-10-23T13:01:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n correlation ID: ef1977d2-31af-4849-9940-4bbd6bbe201a\n client request ID: 363124c1-5480-4435-98da-729155a01144\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n \u003c*errors.joinError | 0x36fd45eaa060\u003e: \n resource group \"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-jdxf2b\" in resourcegroup=\"rg-np-version-upgrade-jdxf2b-lgzkqwrtfnrc\" to finish deleting: GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\",\n \"name\": \"9861ff20-e277-4831-93f1-1162afbd108a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-23T13:01:17.842344425Z\",\n \"endTime\": \"2026-10-23T13:25:21.427910502Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tkg8fusv1drnq89nr4jr4kc118h003a still exists (deletion dispatched at 2026-10-23T13:01:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n \u003cAzure error internals omitted\u003e\n Azure request IDs:\n GET https://rp.j6471040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9861ff20-e277-4831-93f1-1162afbd108a\n correlation ID: ef1977d2-31af-4849-9940-4bbd6bbe201a\n client request ID: 363124c1-5480-4435-98da-729155a01144\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7635/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113590667136471040","occurred_at":"2026-10-23T11:17:38Z","signature_id":"4e8926f9374f6904cc9826c0dee209a2e2cebf4787bc66fa48d1d8df4d4e2067","pr_number":7635,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"1ae9bcc864e0b95b17cfb70f07fc859ef6b54ed4d13d4886751e4ae7fe6a17c0","failure_pattern":"Internal error occurred: failed calling webhook \"webhook","search_query":"Internal error occurred: failed calling webhook \"webhook","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":1,"prior_window_starts":["2026-10-05"],"prior_runs_affected":2,"prior_last_seen_at":"2026-10-07T08:29:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n \u003c*errors.StatusError | 0x157c9dc7bc20\u003e: \n Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"\n {\n ErrStatus: {\n TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n Reason: \"InternalError\",\n Details: {\n Name: \"\",\n Group: \"\",\n Kind: \"\",\n UID: \"\",\n Causes: [\n {\n Type: \"\",\n Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n Field: \"\",\n },\n ],\n RetryAfterSeconds: 0,\n },\n Code: 500,\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:372]: Timed out after 120.003s.\nClusterIssuer CRD should be installed and the resource creatable\nExpected success, but got an error:\n \u003c*errors.StatusError | 0x157c9dc7bc20\u003e: \n Internal error occurred: failed calling webhook \"webhook.cert-manager.io\": failed to call webhook: Post \"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\": no endpoints available for service \"cert-manager-webhook\"\n {\n ErrStatus: {\n TypeMeta: {Kind: \"Status\", APIVersion: \"v1\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n ShardInfo: nil,\n },\n Status: \"Failure\",\n Message: \"Internal error occurred: failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n Reason: \"InternalError\",\n Details: {\n Name: \"\",\n Group: \"\",\n Kind: \"\",\n UID: \"\",\n Causes: [\n {\n Type: \"\",\n Message: \"failed calling webhook \\\"webhook.cert-manager.io\\\": failed to call webhook: Post \\\"https://cert-manager-webhook.cert-manager.svc:443/validate?timeout=30s\\\": no endpoints available for service \\\"cert-manager-webhook\\\"\",\n Field: \"\",\n },\n ],\n RetryAfterSeconds: 0,\n },\n Code: 500,\n },\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7733/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113746348372135936","occurred_at":"2026-10-23T21:36:15Z","signature_id":"ffad9075f8b073e455e1ac724969c29ae57f1247a7dc73dc4b06d2983015219a","pr_number":7733,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0b85ca36ff576f5e5452da4078ad15cf0869fda090c3f787a91acd5ecd81f065","failure_pattern":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","search_query":"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,1,1,1,1,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:182]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0x1025b98de6a8\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0x1025b95ed360\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0x1025b9b81ec0\u003e{\n errs: [\n \u003c*errors.errorString | 0x1025b92c2ec0\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:182]: cluster viability check failed\nExpected success, but got an error:\n \u003c*errors.joinError | 0x1025b98de6a8\u003e: \n VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n {\n errs: [\n \u003c*fmt.wrapError | 0x1025b95ed360\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n err: \u003c*errors.joinError | 0x1025b9b81ec0\u003e{\n errs: [\n \u003c*errors.errorString | 0x1025b92c2ec0\u003e{\n s: \"apiservice/v1beta1.metrics.k8s.io is not available because: failing or missing response from https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1: Get \\\"https://10.128.0.29:10250/apis/metrics.k8s.io/v1beta1\\\": tls: failed to verify certificate: x509: certificate signed by unknown authority\",\n },\n ],\n },\n },\n ],\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113660833035718656","occurred_at":"2026-10-23T15:56:27Z","signature_id":"4b06ceefd21c3c876e41349b56654277c2324b6708e3809cfe51f45fabf75683","pr_number":7073,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"e9b156542f61a22d88e8e787cb5184aae92eaf130a724dfa3a85c4427efdde52","failure_pattern":"VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster \"cluster-hs-pre-7lnp9z\" should complete its version rollout\nUnexpected error:\n \u003c*fmt.wrapError | 0x1290b3e7a900\u003e: \n VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\n {\n msg: \"VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n err: \u003c*verifiers.pollObservationError | 0x1290b3e7a840\u003e{\n observation: \u003c*errors.errorString | 0x1290b436a5d0\u003e{\n s: \"clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n },\n interruption: \u003c*fmt.wrapError | 0x1290b3e7a820\u003e{\n msg: \"failed to get clusterversion \\\"version\\\": context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster \"cluster-hs-pre-7lnp9z\" should complete its version rollout\nUnexpected error:\n \u003c*fmt.wrapError | 0x1290b3e7a900\u003e: \n VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\n {\n msg: \"VerifyClusterVersionRolloutCompleted timed out after 45m0.001s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n err: \u003c*verifiers.pollObservationError | 0x1290b3e7a840\u003e{\n observation: \u003c*errors.errorString | 0x1290b436a5d0\u003e{\n s: \"clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n },\n interruption: \u003c*fmt.wrapError | 0x1290b3e7a820\u003e{\n msg: \"failed to get clusterversion \\\"version\\\": context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7727/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113711841971539968","occurred_at":"2026-10-23T19:19:08Z","signature_id":"06405e4c71e72b70733326b19524a23a628ce5d6d98f1e230f7156ce01e4dc2e","pr_number":7727,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2f3aceed2bd993d066f5fc126ddd12907122d755568207fafbd6490e3839a54a","failure_pattern":"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet \u003cnamespace\u003e/global-pull-secret-syncer has no desired pods scheduled","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0x3221f83f4600\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n {\n msg: \"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n err: \u003c*errors.errorString | 0x3221fa87e2b0\u003e{\n s: \"DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:184]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0x3221f83f4600\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n {\n msg: \"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.009s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n err: \u003c*errors.errorString | 0x3221fa87e2b0\u003e{\n s: \"DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\",\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"8c2d12315219ca1579f718de9c20b9b687edb43eb1f14b0f2771853e17c295f1","pr_number":7463,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"50aafdd0364e960cb215062171251b48e16daefd02feb64915b33b81ee7aa50a","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1df6789587c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*verifiers.pollObservationError | 0x1df6789587a0\u003e{\n observation: \u003c*errors.errorString | 0x1df677e0e5e0\u003e{\n s: \"dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n interruption: \u003c*fmt.wrapError | 0x1df678958780\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1df6776d4500\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1df678958760\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1df679343380\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x1df6789587c0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*verifiers.pollObservationError | 0x1df6789587a0\u003e{\n observation: \u003c*errors.errorString | 0x1df677e0e5e0\u003e{\n s: \"dev/canadacentral/ci01-j0830592-mgmt-2/ocm-arohcpci01-2tkga076iao71irp3jv8khf8go825h67-b5a9t9p3j7h2r9g: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n interruption: \u003c*fmt.wrapError | 0x1df678958780\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1df6776d4500\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x1df678958760\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-mxczzpzdbrzc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T12:28:02.3146282Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x1df679343380\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","signature_id":"7bb7aa38d25f69b831b53681016019396c025b2d18da7932ffb9e298e6579b79","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"af2d21e1e88696d05be0ca90beb20ce563a5bf796115e15a05448c96cb57eced","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x348180f2e220\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*verifiers.pollObservationError | 0x348180f2e200\u003e{\n observation: \u003c*errors.errorString | 0x3481807fdb50\u003e{\n s: \"dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n interruption: \u003c*fmt.wrapError | 0x348180f2e1c0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x3481808801e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x348180f2e1a0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x3481828e4120\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x348180f2e220\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*verifiers.pollObservationError | 0x348180f2e200\u003e{\n observation: \u003c*errors.errorString | 0x3481807fdb50\u003e{\n s: \"dev/centralus/ci01-j9171712-mgmt-1/ocm-arohcpci01-2tkp0dag7nnhqpothvrjnjnqe979h5on-g1t1e6r7l7f2m2p: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n interruption: \u003c*fmt.wrapError | 0x348180f2e1c0\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x3481808801e0\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x348180f2e1a0\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-gh8nb87vhqxg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T22:21:58.5038934Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x3481828e4120\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113744756189171712","occurred_at":"2026-10-23T21:29:55Z","signature_id":"0d5ace1ca3dc48db2d8bb7f3ca58f25eef51b628008732668244b790b4cfd770","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a83a5cad2332d517fe6d82b654adc4ab2b95557c26b786eb1ec2e48ab0c65371","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x23df1c5ad4a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x23df1caefde0\u003e{\n s: \"dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x23df1c5ad4a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-qq9bhv4bt8kw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n err: \u003c*errors.errorString | 0x23df1caefde0\u003e{\n s: \"dev/westus3/ci01-j8250240-mgmt-2/ocm-arohcpci01-2tkbdnpvqhauoe6rgklg00gq6tkhfvrc-r3l8l6v4y8p3s6m: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113511186858250240","occurred_at":"2026-10-23T06:01:48Z","signature_id":"8bc9007fc25b5640ebae9f4863d7359c865853bd666601aeb7d286abdb3e9538","pr_number":7653,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"197f9640ca1cff10bbe506b04538db682332c0dd1edad7858df9b62446a0bf4e","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x794496e4000\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x79449964570\u003e{\n s: \"dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x794496e4000\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-5qtc29s98swx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.003s: dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0x79449964570\u003e{\n s: \"dev/centralus/ci01-j9103104-mgmt-1/ocm-arohcpci01-2tkl21n209ip1b0esf4blj2cm5irkajv-q0k1p4s4p8a4j5a: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113677851789103104","occurred_at":"2026-10-23T17:04:04Z","signature_id":"dfebbf0731f81307daf766e898cda81fae9d9e47bbb68807b280ca90f03dd147","pr_number":7547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"30002276c0d676e1121ab63d27800e924a0d329b41c2b24beb8a2391f90d92df","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x14521ece600\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n err: \u003c*verifiers.pollObservationError | 0x14521ece580\u003e{\n observation: \u003c*errors.errorString | 0x14522933200\u003e{\n s: \"dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n },\n interruption: \u003c*fmt.wrapError | 0x14521ece560\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1452193c140\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x14521ece540\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x14522d0f260\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x14521ece600\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.005s: dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n err: \u003c*verifiers.pollObservationError | 0x14521ece580\u003e{\n observation: \u003c*errors.errorString | 0x14522933200\u003e{\n s: \"dev/centralus/ci01-j1801600-mgmt-1/ocm-arohcpci01-2tkdfmkqdipm175rjcn6kvkhv9opm92k-q4v5p7k2x6m6n2i: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=1 rows=1\",\n },\n interruption: \u003c*fmt.wrapError | 0x14521ece560\u003e{\n msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*errors.Error | 0x1452193c140\u003e{\n Op: 1,\n Kind: 7,\n Err: \u003c*fmt.wrapError | 0x14521ece540\u003e{\n msg: \"With query: let target = \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-bbhmbp7z4kkb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\\\"; let start = datetime(2026-10-23T09:15:12.0541269Z);\\nlet desires = cosmosResourceSnapshots\\n| where resourceID startswith strcat(target, \\\"/\\\")\\n| where resourceType =~ \\\"microsoft.redhatopenshift/hcpopenshiftclusters/readdesires\\\"\\n| extend resourceID = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, resourceID, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, resourceID\\n| where isempty(content.deletionTimestamp)\\n| extend manifest = content.properties.status.kubeContent\\n| where manifest.kind == \\\"HostedCluster\\\" and isempty(manifest.metadata.deletionTimestamp)\\n| extend hostedClusterNamespace = tostring(manifest.metadata.namespace), hostedClusterName = tostring(manifest.metadata.name),\\n managementCluster = tolower(tostring(content.properties.spec.managementCluster))\\n| where isnotempty(hostedClusterNamespace) and isnotempty(hostedClusterName) and isnotempty(managementCluster)\\n| project environment, region, sourceCluster, managementCluster, namespace = strcat(hostedClusterNamespace, \\\"-\\\", hostedClusterName);\\nlet fleet = cosmosResourceSnapshots\\n| where resourceType =~ \\\"microsoft.redhatopenshift/stamps/managementclusters\\\"\\n| extend managementCluster = tolower(resourceID), ts = tolong(content._ts), sourceCluster = cluster\\n| join kind=leftsemi desires on environment, region, sourceCluster, managementCluster\\n| summarize arg_max(timestamp, *) by environment, region, sourceCluster, managementCluster, ts\\n| summarize arg_max(ts, *) by environment, region, sourceCluster, managementCluster\\n| where isempty(content.deletionTimestamp)\\n| extend clusterName = tostring(split(tostring(content.properties.status.aksResourceID), \\\"/\\\")[-1])\\n| where isnotempty(clusterName)\\n| project environment, region, sourceCluster, managementCluster, cluster = clusterName;\\nlet hcp = desires\\n| join kind=inner fleet on environment, region, sourceCluster, managementCluster\\n| distinct environment, region, cluster, namespace;\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project environment, region, cluster, namespace, UID = uid, Node = tostring(object.spec.nodeName);\\nlet counts = pods | summarize Current = count() by environment, region, cluster, namespace;\\nlet deployments = inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project environment, region, cluster, namespace, Desired = tolong(object.spec.replicas);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| extend namespace = tostring(log.pod_namespace), UID = tostring(log.pod_uid)\\n| lookup kind=inner hcp on environment, region, cluster, namespace\\n| summarize arg_max(timestamp, *) by environment, region, cluster, namespace, UID\\n| project environment, region, cluster, namespace, UID, SummaryUID = tostring(log.pod_uid),\\n SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\nhcp\\n| join kind=leftouter deployments on environment, region, cluster, namespace\\n| join kind=leftouter counts on environment, region, cluster, namespace\\n| join kind=leftouter pods on environment, region, cluster, namespace\\n| join kind=leftouter summaries on environment, region, cluster, namespace, UID\\n| project Scope = strcat(environment, \\\"/\\\", region, \\\"/\\\", cluster, \\\"/\\\", namespace),\\n Desired = coalesce(Desired, 0), Current = coalesce(Current, 0), UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n err: \u003c*url.Error | 0x14522d0f260\u003e{\n Op: \"Post\",\n URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n Err: \u003ccontext.deadlineExceededError\u003e{},\n },\n },\n restErrMsg: nil,\n decoded: nil,\n permanent: false,\n inner: nil,\n },\n },\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7709/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113546001531801600","occurred_at":"2026-10-23T08:20:08Z","signature_id":"51444fe40b9218630af21f6e92d01bfeda9c0f66e83382df75ec9d39e17d9e40","pr_number":7709,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f6b19f3be20271a1946ad6e5da34c15339012b8d43c1a7baf3efcab4e3e251f7","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xac144fee4a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0xac1449cf1d0\u003e{\n s: \"dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xac144fee4a0\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-dmsdg7plmb4w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.008s: dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n err: \u003c*errors.errorString | 0xac1449cf1d0\u003e{\n s: \"dev/westus3/ci01-j9121152-mgmt-1/ocm-arohcpci01-2tkofgvg8to4jr06ictqnabr04gnv8tc-i2j7c1e5a0f1n5e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=0 rows=1\",\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113735254979121152","occurred_at":"2026-10-23T20:52:10Z","signature_id":"b819a089f33550631ed5b7c4847276496e869cbd6918d37a9176dd007ad100fd","pr_number":6770,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a346f2ee38025c72bdbab745d0e046d927c6651ff3fe7f95b5953a94c870fd22","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=\u003cuuid\u003e pass=\u003cuuid\u003e: expected healthy complete coverage","search_query":"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x9a09bb1e000\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n err: \u003c*errors.errorString | 0x9a09aa7e480\u003e{\n s: \"router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0x9a09bb1e000\u003e: \n VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\"discovered_targets\":0,\"omitted_targets\":0,\"submitted_targets\":0,\"partial\":true,\"reported_targets\":0,\"outcome\":\"failed\",\"unavailable\":\"discovery_failed\",\"canceled\":false,\"age_seconds\":1300,\"cadence_seconds\":300,\"elapsed_ms\":3,\"roles\":{},\"http_success\":0,\"http_failed\":0,\"tcp_connected\":0,\"tcp_refused\":0,\"tcp_timeout\":0,\"tcp_failed\":0,\"retired_failed\":0,\"dns_success\":0,\"dns_failed\":0,\"dns_reported\":0,\"dns_expected\":0,\"truncated\":0,\"evidence_errors\":0}\n {\n msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-6xjrvnlktzh9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.009s: router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n err: \u003c*errors.errorString | 0x9a09aa7e480\u003e{\n s: \"router=8de50163-6f5a-4470-b7f1-fc49a2127816 pass=08051d34-f874-4cb2-8083-f466a7f470b0: expected healthy complete coverage: {\\\"discovered_targets\\\":0,\\\"omitted_targets\\\":0,\\\"submitted_targets\\\":0,\\\"partial\\\":true,\\\"reported_targets\\\":0,\\\"outcome\\\":\\\"failed\\\",\\\"unavailable\\\":\\\"discovery_failed\\\",\\\"canceled\\\":false,\\\"age_seconds\\\":1300,\\\"cadence_seconds\\\":300,\\\"elapsed_ms\\\":3,\\\"roles\\\":{},\\\"http_success\\\":0,\\\"http_failed\\\":0,\\\"tcp_connected\\\":0,\\\"tcp_refused\\\":0,\\\"tcp_timeout\\\":0,\\\"tcp_failed\\\":0,\\\"retired_failed\\\":0,\\\"dns_success\\\":0,\\\"dns_failed\\\":0,\\\"dns_reported\\\":0,\\\"dns_expected\\\":0,\\\"truncated\\\":0,\\\"evidence_errors\\\":0}\",\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"7dc8c3eafac5404affbb8c6a5aceaf23d8664d06a783b6429405a361c74eae60","pr_number":7463,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":86,"trend_counts":[0,0,6,12,8,2,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":262,"prior_last_seen_at":"2026-10-16T16:35:11Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T08:40:51Z\n Ended: 2026-10-23T08:58:52Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1650944-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-23T08:40:51Z\n Ended: 2026-10-23T08:57:52Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1650944-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113536694941650944","occurred_at":"2026-10-23T07:43:10Z","signature_id":"aeb0e7cf0f7fb6668a74127ede7587f34beff19ae9034d243e6f7865af3d860f","pr_number":7073,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,6,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":26,"prior_last_seen_at":"2026-10-15T19:10:09Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T12:46:11Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T12:46:11Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","signature_id":"91e41adf8a4cbf20601080ac6e87fc658cc44668486f116ba1401163a81cca7b","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,2,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":14,"prior_last_seen_at":"2026-10-15T19:10:09Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T12:56:11Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T12:56:11Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0830592-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.252:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0830592/providers/microsoft.monitor/accounts/services-j0830592\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-g9k4t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113592791710830592","occurred_at":"2026-10-23T11:26:04Z","signature_id":"7467c8b82e1de4741264fd7a83a7e3d54e11cebdb1e11c074699979a6d8ec2ff","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":11,"trend_counts":[0,0,0,1,0,4,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-09-21","2026-09-28","2026-10-05","2026-10-12"],"prior_runs_affected":62,"prior_last_seen_at":"2026-10-16T14:51:17Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 62 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-q7xh2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-q7xh2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-ndlbl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-ndlbl has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-7xjfm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-7xjfm has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-5dvf5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-5dvf5 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-mb68l\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-mb68l has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-ssk4m\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-ssk4m has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-9ljfg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-9ljfg has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-hlrms\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-hlrms has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-7zxmv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-7zxmv has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-995kh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-995kh has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-jj844\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-jj844 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-hk2pc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-hk2pc has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-v22fj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-v22fj has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-x55nd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-x55nd has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-dbxr7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-dbxr7 has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-gmslj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-gmslj has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-tfmkl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-tfmkl has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-wcfrc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-wcfrc has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-vz2cv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-vz2cv has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-9mcmc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-9mcmc has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-d6nhd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-d6nhd has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-9txxf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-9txxf has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-6b8wc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-6b8wc has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-nrgxf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-nrgxf has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-sgtgx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-sgtgx has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-rqj2v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-rqj2v has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-8vhqg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-8vhqg has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-rsscm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-rsscm has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-t4fln\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-t4fln has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-tjxjb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-tjxjb has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-m5d5c\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-m5d5c has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-bhfw8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-bhfw8 has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-8kc57\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-8kc57 has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-nmlxd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-nmlxd has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-7nz9z\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-7nz9z has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-rr5tb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-rr5tb has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-zgzmg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-zgzmg has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-v2xpg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-v2xpg has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-v96bl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-v96bl has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-j9jl5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-j9jl5 has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-b87xs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-b87xs has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-pphfm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-pphfm has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-hgkjq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-hgkjq has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-xd72x\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-xd72x has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-8jcjf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-8jcjf has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-rvdns\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-rvdns has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-x4kzj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-x4kzj has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-b8qd2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-b8qd2 has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-qt827\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-qt827 has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-q8mvg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-q8mvg has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-kfjrt\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-kfjrt has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-hwkjz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-hwkjz has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-9td26\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-9td26 has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-cr96z\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-cr96z has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-9dqr2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-9dqr2 has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-d56c6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-d56c6 has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-s89qq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-s89qq has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-k2xxx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-k2xxx has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-jb7dj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-jb7dj has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-l6zq2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-l6zq2 has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-hzddl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-hzddl has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Fired (not resolved)\n Started: 2026-10-23T14:33:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5183488-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-67995d8d8b-k2wfc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-67995d8d8b-k2wfc has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7547/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113627585765183488","occurred_at":"2026-10-23T13:44:20Z","signature_id":"28d90db273f3dc28418a9638298084d32843c4ff21548aa85701ba2afb5f93dc","pr_number":7547,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,0,0,1,1,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-21","2026-10-05","2026-10-12"],"prior_runs_affected":23,"prior_last_seen_at":"2026-10-16T14:51:17Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-23T02:40:08Z\n Ended: 2026-10-23T02:49:10Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j3977984-mgmt-2\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113446671063977984","occurred_at":"2026-10-23T01:45:26Z","signature_id":"7d7d5f484ae9cf25602860243ff6931ef9c6211bb6bd55469a3f901426372f60","pr_number":7662,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24f84e8da288b18c5a6ce017a06862a7753a25291f7369a28e80acece5813bd2","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2b8958bd95b0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.004s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0x2b8958bd95b0\u003e: \n missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\n {\n s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per source, missing: [acm-agent]; observed counts: maestro=1115, hypershift=13672, acm-agent=0\",\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113701532795932672","occurred_at":"2026-10-23T18:38:10Z","signature_id":"97dc8f8152c396287fe01fe00d78417ce23034d6cc4cad15e28215178e3bc6eb","pr_number":7712,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have at least 2 nodes (autoscaling min), found 0","search_query":"node pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":1.2820512820512822,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,1],"trend_range":"2026-10-17..2026-10-23 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.004s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: 0\nto be \u003e=\n \u003cint\u003e: 2\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:519]: Timed out after 900.004s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:506 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: 0\nto be \u003e=\n \u003cint\u003e: 2"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2113501053063270400","occurred_at":"2026-10-23T05:21:32Z","signature_id":"93484e969eefe4c20a3e175e375caad69b1f31f45ea3188e5adb186dca3ed1fc","pr_number":7463,"after_last_push_of_merged_pr":true}]}]}]}