{"meta":{"date":"2026-10-01","timezone":"UTC","generated_at":"2026-10-29T22:18:54Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105795013597204480","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-01T23:00:29Z","started_at":"2026-10-01T23:00:29Z","completed_at":"2026-10-02T00:02:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T00:13:18.053053989Z","timing_metadata_checked_at":"2026-10-02T00:13:18.053053989Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-02T00:13:18.060791865Z","region_metadata_checked_at":"2026-10-02T01:17:03.295816955Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d4f19f6bcb901e70e42ea6eea55837cbaf183023e554f2d549a1c5b8370aa319","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105795013597204480","occurred_at":"2026-10-01T23:00:29Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-01T22:19:29Z","started_at":"2026-10-01T22:19:29Z","completed_at":"2026-10-02T00:48:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T01:06:31.44184389Z","timing_metadata_checked_at":"2026-10-02T01:06:31.44184389Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T01:06:31.444377981Z","region_metadata_checked_at":"2026-10-02T01:06:31.444377981Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"0613116356995f61223b434eb22523c9ddb30b6a78aa8c19fa9eea354855e81a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"261d2ba977979da2043d1d05e7fda09b0050628d435b4e8f6c70c65d7ede0f72","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001012e40\u003e: \n failed to create NodePool np-6g6tzd: failed waiting for nodepool=\"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\" in resourcegroup=\"rg-candidate-4-22-xz7x2977wmz2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n \"name\": \"3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:22:22.461339415Z\",\n \"endTime\": \"2026-10-01T23:41:27.247269633Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: Creating: virtualmachine creating or updating\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001012e40\u003e: \n failed to create NodePool np-6g6tzd: failed waiting for nodepool=\"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\" in resourcegroup=\"rg-candidate-4-22-xz7x2977wmz2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n \"name\": \"3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:22:22.461339415Z\",\n \"endTime\": \"2026-10-01T23:41:27.247269633Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: Creating: virtualmachine creating or updating\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"14572c9cba012c49beb967105a553e0b8bbe3a244a4b26657bd8a813b4b0535a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"153a3254e17592610cd68e651bfb6cbf2394189d89044802b8c3b533e149203a","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f50a20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-6svllmjvpbpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\",\n \"name\": \"7c5e553a-2bca-431a-819c-94122ad78b6b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T00:04:50.703195637Z\",\n \"endTime\": \"2026-10-02T00:23:53.181881109Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f50a20\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-6svllmjvpbpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\",\n \"name\": \"7c5e553a-2bca-431a-819c-94122ad78b6b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T00:04:50.703195637Z\",\n \"endTime\": \"2026-10-02T00:23:53.181881109Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"43b12b70c0a9497a6579d34b5f016908eff215cfbcac68eaf6f41e0ee8689ba1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"577a2f2cdd3de8213f2886c7670e9678f9d789b1a765ea47d5053e8926ce155f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-58mkpz\" for cluster \"cluster-candidate-4-21-58mkpz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cee380\u003e: \n failed to create NodePool np-58mkpz: failed waiting for nodepool=\"np-58mkpz\" for cluster \"cluster-candidate-4-21-58mkpz\" in resourcegroup=\"rg-candidate-4-21-8w4nf2bg8hwt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/22afbbb9-6f81-4f16-a70d-aa16064ca2b5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/22afbbb9-6f81-4f16-a70d-aa16064ca2b5\",\n \"name\": \"22afbbb9-6f81-4f16-a70d-aa16064ca2b5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:22:00.007708534Z\",\n \"endTime\": \"2026-10-01T23:41:07.215796872Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k3l1b7w5a7b5w3u-np-58mkpz-kgdz9-w8q2v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k3l1b7w5a7b5w3u-np-58mkpz-kgdz9-xfhks: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k3l1b7w5a7b5w3u-np-58mkpz-kgdz9-w8q2v: Creating: virtualmachine creating or updating\\nMachine k3l1b7w5a7b5w3u-np-58mkpz-kgdz9-xfhks: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-58mkpz\" for cluster \"cluster-candidate-4-21-58mkpz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cee380\u003e: \n failed to create NodePool np-58mkpz: failed waiting for nodepool=\"np-58mkpz\" for cluster \"cluster-candidate-4-21-58mkpz\" in resourcegroup=\"rg-candidate-4-21-8w4nf2bg8hwt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/22afbbb9-6f81-4f16-a70d-aa16064ca2b5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/22afbbb9-6f81-4f16-a70d-aa16064ca2b5\",\n \"name\": \"22afbbb9-6f81-4f16-a70d-aa16064ca2b5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:22:00.007708534Z\",\n \"endTime\": \"2026-10-01T23:41:07.215796872Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k3l1b7w5a7b5w3u-np-58mkpz-kgdz9-w8q2v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k3l1b7w5a7b5w3u-np-58mkpz-kgdz9-xfhks: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k3l1b7w5a7b5w3u-np-58mkpz-kgdz9-w8q2v: Creating: virtualmachine creating or updating\\nMachine k3l1b7w5a7b5w3u-np-58mkpz-kgdz9-xfhks: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4d0b57211481d4b9285dda3d2c01f9c63dc431967f4ab7cffb2e9a242a9f0525","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"84ad30e1bfed5ae9658a5da88f0ca1af31ae55f3c693883c462c46dcf7608e01","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nz54xv\" for cluster \"cluster-candidate-4-23-nz54xv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dbc340\u003e: \n failed to create NodePool np-nz54xv: failed waiting for nodepool=\"np-nz54xv\" for cluster \"cluster-candidate-4-23-nz54xv\" in resourcegroup=\"rg-candidate-4-23-87g9stccjxzb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7507c59e-c9ca-4318-bcfd-f4b2fc2e72d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7507c59e-c9ca-4318-bcfd-f4b2fc2e72d5\",\n \"name\": \"7507c59e-c9ca-4318-bcfd-f4b2fc2e72d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:22:46.805553224Z\",\n \"endTime\": \"2026-10-01T23:41:47.237736739Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d9g4u8w0a7x2b4h-np-nz54xv-qb8xb-q48tw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine d9g4u8w0a7x2b4h-np-nz54xv-qb8xb-qpx5x: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d9g4u8w0a7x2b4h-np-nz54xv-qb8xb-q48tw: Creating: virtualmachine creating or updating\\nMachine d9g4u8w0a7x2b4h-np-nz54xv-qb8xb-qpx5x: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nz54xv\" for cluster \"cluster-candidate-4-23-nz54xv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dbc340\u003e: \n failed to create NodePool np-nz54xv: failed waiting for nodepool=\"np-nz54xv\" for cluster \"cluster-candidate-4-23-nz54xv\" in resourcegroup=\"rg-candidate-4-23-87g9stccjxzb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7507c59e-c9ca-4318-bcfd-f4b2fc2e72d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7507c59e-c9ca-4318-bcfd-f4b2fc2e72d5\",\n \"name\": \"7507c59e-c9ca-4318-bcfd-f4b2fc2e72d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:22:46.805553224Z\",\n \"endTime\": \"2026-10-01T23:41:47.237736739Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d9g4u8w0a7x2b4h-np-nz54xv-qb8xb-q48tw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine d9g4u8w0a7x2b4h-np-nz54xv-qb8xb-qpx5x: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d9g4u8w0a7x2b4h-np-nz54xv-qb8xb-q48tw: Creating: virtualmachine creating or updating\\nMachine d9g4u8w0a7x2b4h-np-nz54xv-qb8xb-qpx5x: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"600e6b8016554ca10682f623d9e712cf4dd8bade2f51f304f019fe46a8e0b5fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"b4159087ef39a5abb0c42cbb58f2ea45413e0c9d5e74cdc7078f52b05a7b45c5","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a47540\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-rj7chpbn7z2c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dd4c07c5-fe2d-47bf-9ef6-ea2171664623\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dd4c07c5-fe2d-47bf-9ef6-ea2171664623\",\n \"name\": \"dd4c07c5-fe2d-47bf-9ef6-ea2171664623\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T22:58:07.666255851Z\",\n \"endTime\": \"2026-10-01T23:17:14.973053483Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-9d89g-jglhd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-9d89g-jglhd: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a47540\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-rj7chpbn7z2c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dd4c07c5-fe2d-47bf-9ef6-ea2171664623\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dd4c07c5-fe2d-47bf-9ef6-ea2171664623\",\n \"name\": \"dd4c07c5-fe2d-47bf-9ef6-ea2171664623\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T22:58:07.666255851Z\",\n \"endTime\": \"2026-10-01T23:17:14.973053483Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-9d89g-jglhd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-9d89g-jglhd: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"63f016a0d26d50acf7b1c5b443f2a8d09976c42b8ce8b3744498080a13a7a895","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"bdfd444f6b247ef48624113640b671ccd50695d691f9d049b710ad7273122875","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8gxplz\" for cluster \"cluster-candidate-5-1-8gxplz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000acaa40\u003e: \n failed to create NodePool np-8gxplz: failed waiting for nodepool=\"np-8gxplz\" for cluster \"cluster-candidate-5-1-8gxplz\" in resourcegroup=\"rg-candidate-5-1-kvc5d4thszfq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bde76db6-b7ac-422e-8660-ba8904affbd3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bde76db6-b7ac-422e-8660-ba8904affbd3\",\n \"name\": \"bde76db6-b7ac-422e-8660-ba8904affbd3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:26:31.681894722Z\",\n \"endTime\": \"2026-10-01T23:45:37.242801753Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o5t7r1t3k4e4s3h-np-8gxplz-bxndq-245qj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o5t7r1t3k4e4s3h-np-8gxplz-bxndq-2577p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o5t7r1t3k4e4s3h-np-8gxplz-bxndq-245qj: Creating: virtualmachine creating or updating\\nMachine o5t7r1t3k4e4s3h-np-8gxplz-bxndq-2577p: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8gxplz\" for cluster \"cluster-candidate-5-1-8gxplz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000acaa40\u003e: \n failed to create NodePool np-8gxplz: failed waiting for nodepool=\"np-8gxplz\" for cluster \"cluster-candidate-5-1-8gxplz\" in resourcegroup=\"rg-candidate-5-1-kvc5d4thszfq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bde76db6-b7ac-422e-8660-ba8904affbd3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bde76db6-b7ac-422e-8660-ba8904affbd3\",\n \"name\": \"bde76db6-b7ac-422e-8660-ba8904affbd3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:26:31.681894722Z\",\n \"endTime\": \"2026-10-01T23:45:37.242801753Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o5t7r1t3k4e4s3h-np-8gxplz-bxndq-245qj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o5t7r1t3k4e4s3h-np-8gxplz-bxndq-2577p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o5t7r1t3k4e4s3h-np-8gxplz-bxndq-245qj: Creating: virtualmachine creating or updating\\nMachine o5t7r1t3k4e4s3h-np-8gxplz-bxndq-2577p: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7515828470e1d62b2a52a289358f1abb69d66ae8b8f57796d8f7087457535829","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"307bc38fc0da1536f942492cf74c8e559878301b0fc5629d37255c20e9ee7ec4","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019901c8\u003e: \n resource group \"oidc-wi-nq9xg56z8ptc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nq9xg56z8ptc\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n \"name\": \"b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T00:11:10.101784102Z\",\n \"endTime\": \"2026-10-02T00:35:13.452817702Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t6a9r7piu3mrhnemk9adlcvlla5mb7v still exists (deletion dispatched at 2026-10-02T00:11:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019901c8\u003e: \n resource group \"oidc-wi-nq9xg56z8ptc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nq9xg56z8ptc\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n \"name\": \"b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T00:11:10.101784102Z\",\n \"endTime\": \"2026-10-02T00:35:13.452817702Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t6a9r7piu3mrhnemk9adlcvlla5mb7v still exists (deletion dispatched at 2026-10-02T00:11:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"818f2a7bd407ba132de89791c4d9b60c8424f506c7d3c644b1a749dd16face22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"bff4c5f332cfa283a8daa21713459950af8584404733577995b07a2a2bfcc5a8","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000d82400\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-g79pk8vm4jx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n \"name\": \"d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:53:42.212331844Z\",\n \"endTime\": \"2026-10-02T00:12:43.153935456Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: Creating: virtualmachine creating or updating\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000d82400\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-g79pk8vm4jx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n \"name\": \"d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:53:42.212331844Z\",\n \"endTime\": \"2026-10-02T00:12:43.153935456Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: Creating: virtualmachine creating or updating\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-g79pk8vm4jx6\" to finish creating: GET https://management"}},{"row_id":"84a2048d9e24e3ef68295390709fa8265c028a9d640c3f7c39422a039d6f5d22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"2169238a3efc65d2f002e87d4107ee839610a68181a3b21a5f5f8555c8c0152b","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001f200c0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4rdzlcbp8jqd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\",\n \"name\": \"7941e9fc-eabb-4912-b47f-1619c832d931\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:02:15.574287096Z\",\n \"endTime\": \"2026-10-01T23:21:15.149755975Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001f200c0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4rdzlcbp8jqd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\",\n \"name\": \"7941e9fc-eabb-4912-b47f-1619c832d931\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:02:15.574287096Z\",\n \"endTime\": \"2026-10-01T23:21:15.149755975Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cf312d6de260db493dfca094faeb1e6b598be51bac8d4e4f074898b89952fe3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"6fc83d6684638cb567013759a7193f362c34949e41b3a8668c62f64a251d057c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a16440\u003e: \n failed to create NodePool np-k96n75: failed waiting for nodepool=\"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\" in resourcegroup=\"rg-candidate-5-0-gclccxznszk7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n \"name\": \"e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:23:36.842890739Z\",\n \"endTime\": \"2026-10-01T23:42:37.244899482Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: Creating: virtualmachine creating or updating\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a16440\u003e: \n failed to create NodePool np-k96n75: failed waiting for nodepool=\"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\" in resourcegroup=\"rg-candidate-5-0-gclccxznszk7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n \"name\": \"e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:23:36.842890739Z\",\n \"endTime\": \"2026-10-01T23:42:37.244899482Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: Creating: virtualmachine creating or updating\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e39a8d4f6a415e6c09d5d3ce317c0cc46b17ef6069ac556bead6a53233c1a021","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"951b0a5c5a70e3b35bb3f64267eb4dc0e2b6817b2864d1a40fa29654b0ff2a81","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001c16198\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-mrv84pkzs8gp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n \"name\": \"c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:02:38.499943334Z\",\n \"endTime\": \"2026-10-01T23:21:45.00235976Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001c16198\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-mrv84pkzs8gp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n \"name\": \"c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:02:38.499943334Z\",\n \"endTime\": \"2026-10-01T23:21:45.00235976Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"eee8270cd1593c7a6038ff9c3b76016ec291b3bf9453be131d5f975171a66a26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"4247285a4a7e0ce74bb11c7155b281ec95aa3b9b5e58b9fb3220d4d61dc786b2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bl9ftv\" for cluster \"cluster-candidate-4-20-bl9ftv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001076a40\u003e: \n failed to create NodePool np-bl9ftv: failed waiting for nodepool=\"np-bl9ftv\" for cluster \"cluster-candidate-4-20-bl9ftv\" in resourcegroup=\"rg-candidate-4-20-x4gs2n6gstps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c10fd3d9-c2fc-4d77-bb0e-8b5b7e8cde2c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c10fd3d9-c2fc-4d77-bb0e-8b5b7e8cde2c\",\n \"name\": \"c10fd3d9-c2fc-4d77-bb0e-8b5b7e8cde2c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:21:31.953266163Z\",\n \"endTime\": \"2026-10-01T23:40:37.297291005Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i3l0t0t0y7o2p4y-np-bl9ftv-t8xwv-4tjkc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine i3l0t0t0y7o2p4y-np-bl9ftv-t8xwv-pjccm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i3l0t0t0y7o2p4y-np-bl9ftv-t8xwv-4tjkc: Creating: virtualmachine creating or updating\\nMachine i3l0t0t0y7o2p4y-np-bl9ftv-t8xwv-pjccm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bl9ftv\" for cluster \"cluster-candidate-4-20-bl9ftv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001076a40\u003e: \n failed to create NodePool np-bl9ftv: failed waiting for nodepool=\"np-bl9ftv\" for cluster \"cluster-candidate-4-20-bl9ftv\" in resourcegroup=\"rg-candidate-4-20-x4gs2n6gstps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c10fd3d9-c2fc-4d77-bb0e-8b5b7e8cde2c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c10fd3d9-c2fc-4d77-bb0e-8b5b7e8cde2c\",\n \"name\": \"c10fd3d9-c2fc-4d77-bb0e-8b5b7e8cde2c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:21:31.953266163Z\",\n \"endTime\": \"2026-10-01T23:40:37.297291005Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i3l0t0t0y7o2p4y-np-bl9ftv-t8xwv-4tjkc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine i3l0t0t0y7o2p4y-np-bl9ftv-t8xwv-pjccm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i3l0t0t0y7o2p4y-np-bl9ftv-t8xwv-4tjkc: Creating: virtualmachine creating or updating\\nMachine i3l0t0t0y7o2p4y-np-bl9ftv-t8xwv-pjccm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ef1e7bc19c0be55b458582de5afb564b01903f8e76870d15f9b0fefa09ea648a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","failed_at":"e2e","signature_id":"1b39a1fcea6ce322dd8fa1bd054343dbd452f0f0491cacc4f8b9ef7b6a0493be","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009252c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-tfpfzw4vzd4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\",\n \"name\": \"8e650b8c-c903-48db-ae29-20e1c0f74604\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:01:41.16595227Z\",\n \"endTime\": \"2026-10-01T23:20:45.151714689Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: Creating: virtualmachine creating or updating\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009252c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-tfpfzw4vzd4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\",\n \"name\": \"8e650b8c-c903-48db-ae29-20e1c0f74604\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T23:01:41.16595227Z\",\n \"endTime\": \"2026-10-01T23:20:45.151714689Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: Creating: virtualmachine creating or updating\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-01T02:00:28Z","started_at":"2026-10-01T02:00:28Z","completed_at":"2026-10-01T04:34:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T04:46:58.421080406Z","timing_metadata_checked_at":"2026-10-01T04:46:58.421080406Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T04:46:58.424300195Z","region_metadata_checked_at":"2026-10-01T04:46:58.424300195Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"4ac8d979fe7deea312c77b0dce827e1972d535337f509ece100f3ab6400164b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"5565e5f16c645b130d162f8ebf645832f59d83ad60615990833bdafaca4b6ef8","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-c4cdqr\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00120fcb0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-bpfkv76t7qfk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-c4cdqr\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-c4cdqr\" to 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00120fcb0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-bpfkv76t7qfk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-c4cdqr\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"4e25ed8c3e0ae243204fae47c718544822f4a7949ae9b03042347d8f00aba1a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"fd3be8a2cabe6e92eed1d71ec31a986980accf6d34064931d1b31fd3558ce497","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-5-0-kwx4gf\" to follow minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014a3d10\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-l8ml659lgnj5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-kwx4gf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-5-0-kwx4gf\" to follow minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014a3d10\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-l8ml659lgnj5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-kwx4gf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"518229db34afbfd6aecf7288056ec93f7fc645f089d7b5f61c98a330a3c0fda1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"f03158936394395704c5b6ff64b4370dd9e2548b869de7e69671c413ff75621f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-bngfn7\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000cf5770\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-q9xpw68pgwb7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-bngfn7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-bngfn7\" to 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000cf5770\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-q9xpw68pgwb7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-bngfn7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"736f5f87408c04504da5bffcec7571c7d900a9bd0bfedeaa6660ba528df646ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"a8eb2d795a11c6dd632258debe4cb2cff59cae61bb73a703f5d0ea485effd6ec","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-20-222nr2\" to follow minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000993530\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-vf9snh5jjb8g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-222nr2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-20-222nr2\" to follow minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000993530\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-vf9snh5jjb8g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-222nr2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"8d70e002860b3d7d38b5d01c42bca88d024c176d3cc9c6c3af0ff44fd21d249c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"623dfd830d068e8fc297a04ee4a56d7cb6e4821c0251af7874f2b77ba7c26fa6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00081f300\u003e: \n failed to create NodePool np-8l66wr: failed waiting for nodepool=\"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\" in resourcegroup=\"rg-candidate-4-20-66r4zk7wm6rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\",\n \"name\": \"1513980f-1a90-4c5b-865f-c0a3b6601404\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T03:20:14.931117191Z\",\n \"endTime\": \"2026-10-01T03:39:22.388104494Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00081f300\u003e: \n failed to create NodePool np-8l66wr: failed waiting for nodepool=\"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\" in resourcegroup=\"rg-candidate-4-20-66r4zk7wm6rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\",\n \"name\": \"1513980f-1a90-4c5b-865f-c0a3b6601404\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T03:20:14.931117191Z\",\n \"endTime\": \"2026-10-01T03:39:22.388104494Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"94f5f0078b18e922c85d43ada199a0daebc3340be05277a883c06942b8c25680","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"2388c2f6af983771920940504bcf433d725fb8c7cc1e81c55f6ab467636ed7ef","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-sq7dhp\" to follow minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015c4e70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-vjpbzl2jbr62/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-sq7dhp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-sq7dhp\" to follow minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015c4e70\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-vjpbzl2jbr62/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-sq7dhp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"98cd1452b4f6dff913964978b46ce696096709376fb3d856d6bd04079a4b48d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"f47c5337335a014c2bed2f388006a403bbb00ccd35a477f73d5472519ff75085","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00171ed20\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-kc5k44mb955m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdbbe915-46e7-47c2-be0d-e2cbc376488e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdbbe915-46e7-47c2-be0d-e2cbc376488e\",\n \"name\": \"bdbbe915-46e7-47c2-be0d-e2cbc376488e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T02:58:20.214032414Z\",\n \"endTime\": \"2026-10-01T03:17:20.370385488Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-wwgqk-cz84n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-kc5k44mb955m--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-wwgqk-cz84n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-wwgqk-cz84n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-kc5k44mb955m--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-wwgqk-cz84n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00171ed20\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-kc5k44mb955m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdbbe915-46e7-47c2-be0d-e2cbc376488e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bdbbe915-46e7-47c2-be0d-e2cbc376488e\",\n \"name\": \"bdbbe915-46e7-47c2-be0d-e2cbc376488e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T02:58:20.214032414Z\",\n \"endTime\": \"2026-10-01T03:17:20.370385488Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-wwgqk-cz84n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-kc5k44mb955m--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-wwgqk-cz84n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-wwgqk-cz84n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-kc5k44mb955m--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-wwgqk-cz84n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b2a74a49dfe32da057cd34b8ecbadc3c2af99d260629521eab67fa87add76054","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"1719e4976d13747a5ba884e6917c1488f67a8a955b14f27daff4e13ed2902037","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-hwc6dp\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00164d620\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-txlxhwglc5dq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-hwc6dp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-hwc6dp\" to 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00164d620\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-txlxhwglc5dq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-hwc6dp\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"be215448f72f0db044dcacc8a7128360cca3b527395a037977dafd182068f346","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"844c1e8cf0cc01d92e68b7c7011864371b606ef8f2d55622248024c05d7ce83e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-22-x5rqn9\" to follow minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00163b1d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-d779fckrdm2l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-x5rqn9\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-22-x5rqn9\" to follow minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00163b1d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-d779fckrdm2l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-x5rqn9\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"cbe909355c89dcc194321f189bc1e53823861b27f78c21a489459cc5af16b373","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"11a7f777efd16a4602f165155402df79554a160ce761b48987b2fa81eaa853ff","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-kbwzzs\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00104fe60\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-qb6v87bszxhb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-kbwzzs\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-kbwzzs\" to 5.1\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00104fe60\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-qb6v87bszxhb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-kbwzzs\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"da5602e08c32d61efa6cd879d699380221c7b45ce8035a6800502cf40f77a1cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"c14433cba7a1a57c209cdd5665919f16d75f3d85ba2502e0034b92b8f4aee701","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-psr6l5\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001800d20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-rn244qh65ldr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-psr6l5\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-psr6l5\" to 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001800d20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-rn244qh65ldr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-psr6l5\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"e17cd62a231999c066f123b0245cca50bfaa9e9d9893618c0802a41816a504e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"a09d9f975ede859b703696d2a88baac07d837fd9c31b0ebe46690181381da93a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-21-pr7dlf\" to follow minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013e6d20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4ds2lzg2vct6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-pr7dlf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-21-pr7dlf\" to follow minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013e6d20\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-4ds2lzg2vct6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-pr7dlf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidNullTagValue\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidNullTagValue\",\n \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue"}},{"row_id":"e9052bfd6bf2563761dd35922f817e60b700a130429dbe3835133489f10ca995","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","failed_at":"e2e","signature_id":"c574b07b3114ca80e839d7f5d60517775ee576de2d943538a90489d57cb3eec5","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6ac40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-b7k2xmx45mmg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f67a4120-58f4-48a2-b9dd-9ee7da78a289\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f67a4120-58f4-48a2-b9dd-9ee7da78a289\",\n \"name\": \"f67a4120-58f4-48a2-b9dd-9ee7da78a289\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T02:57:30.013121006Z\",\n \"endTime\": \"2026-10-01T03:16:30.182282415Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h5u8t2s6s2i7w5q-np-1-78d96-vn4sd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-b7k2xmx45mmg--managed/providers/Microsoft.Compute/virtualMachines/h5u8t2s6s2i7w5q-np-1-78d96-vn4sd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h5u8t2s6s2i7w5q-np-1-78d96-vn4sd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-b7k2xmx45mmg--managed/providers/Microsoft.Compute/virtualMachines/h5u8t2s6s2i7w5q-np-1-78d96-vn4sd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d6ac40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-b7k2xmx45mmg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f67a4120-58f4-48a2-b9dd-9ee7da78a289\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f67a4120-58f4-48a2-b9dd-9ee7da78a289\",\n \"name\": \"f67a4120-58f4-48a2-b9dd-9ee7da78a289\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T02:57:30.013121006Z\",\n \"endTime\": \"2026-10-01T03:16:30.182282415Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h5u8t2s6s2i7w5q-np-1-78d96-vn4sd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-b7k2xmx45mmg--managed/providers/Microsoft.Compute/virtualMachines/h5u8t2s6s2i7w5q-np-1-78d96-vn4sd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h5u8t2s6s2i7w5q-np-1-78d96-vn4sd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-b7k2xmx45mmg--managed/providers/Microsoft.Compute/virtualMachines/h5u8t2s6s2i7w5q-np-1-78d96-vn4sd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105450319994425344","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-01T00:10:48Z","started_at":"2026-10-01T00:10:48Z","completed_at":"2026-10-01T02:44:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T02:52:11.558445077Z","timing_metadata_checked_at":"2026-10-01T02:52:11.558445077Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T02:52:11.561962066Z","region_metadata_checked_at":"2026-10-01T02:52:11.561962066Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5cc1ce8ec7670cb3ce427652622659a52812a746ad468f8837db8d2e081f8023","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105450319994425344","occurred_at":"2026-10-01T00:10:48Z","failed_at":"e2e","signature_id":"b47c99754a6a1f045aa9bf003dd005e8d53c7b6f39de576acac3bb76b9fadb06","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000884ae0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gcjj2967bz4p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d64d0e1-6be5-43b3-88bc-d71f31dca2c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d64d0e1-6be5-43b3-88bc-d71f31dca2c3\",\n \"name\": \"8d64d0e1-6be5-43b3-88bc-d71f31dca2c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T01:34:57.093691985Z\",\n \"endTime\": \"2026-10-01T01:54:01.709819246Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine z9s5e8x2m9t9g2t-autoscale-noaz-88jfb-tfvcz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gcjj2967bz4p--managed/providers/Microsoft.Compute/virtualMachines/z9s5e8x2m9t9g2t-autoscale-noaz-88jfb-tfvcz to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine z9s5e8x2m9t9g2t-autoscale-noaz-88jfb-tfvcz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gcjj2967bz4p--managed/providers/Microsoft.Compute/virtualMachines/z9s5e8x2m9t9g2t-autoscale-noaz-88jfb-tfvcz to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000884ae0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gcjj2967bz4p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d64d0e1-6be5-43b3-88bc-d71f31dca2c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d64d0e1-6be5-43b3-88bc-d71f31dca2c3\",\n \"name\": \"8d64d0e1-6be5-43b3-88bc-d71f31dca2c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-01T01:34:57.093691985Z\",\n \"endTime\": \"2026-10-01T01:54:01.709819246Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine z9s5e8x2m9t9g2t-autoscale-noaz-88jfb-tfvcz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gcjj2967bz4p--managed/providers/Microsoft.Compute/virtualMachines/z9s5e8x2m9t9g2t-autoscale-noaz-88jfb-tfvcz to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine z9s5e8x2m9t9g2t-autoscale-noaz-88jfb-tfvcz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-gcjj2967bz4p--managed/providers/Microsoft.Compute/virtualMachines/z9s5e8x2m9t9g2t-autoscale-noaz-88jfb-tfvcz to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}