{"meta":{"date":"2026-10-03","timezone":"UTC","generated_at":"2026-10-29T20:20:13Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T23:00:45Z","started_at":"2026-10-03T23:00:45Z","completed_at":"2026-10-04T00:25:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-04T00:26:41.290696754Z","timing_metadata_checked_at":"2026-10-04T00:26:41.290696754Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-04T00:26:41.293780245Z","region_metadata_checked_at":"2026-10-04T00:26:41.293780245Z"},"failed_at":["e2e"],"failed_test_count":14,"occurrences":[{"row_id":"07615d8874a0cfc8bd62083e1b2a26b8abc1ce63b444a08aade6f3ff35a7a270","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"d2485400b9fb14ab84f4a84285253177801b00cc9d4149c5818d0c29ddef9be2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x3731739b0018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4h58pvg4tw2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1659164e-7252-4678-9399-950feff9f29d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1659164e-7252-4678-9399-950feff9f29d\",\n \"name\": \"1659164e-7252-4678-9399-950feff9f29d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:15:46.648351931Z\",\n \"endTime\": \"2026-10-03T23:34:50.305129886Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-hkqzd-6mfw6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-hkqzd-7x9wt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-hkqzd-6mfw6: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-hkqzd-7x9wt: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x3731739b0018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4h58pvg4tw2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1659164e-7252-4678-9399-950feff9f29d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1659164e-7252-4678-9399-950feff9f29d\",\n \"name\": \"1659164e-7252-4678-9399-950feff9f29d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:15:46.648351931Z\",\n \"endTime\": \"2026-10-03T23:34:50.305129886Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-hkqzd-6mfw6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-hkqzd-7x9wt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-hkqzd-6mfw6: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-hkqzd-7x9wt: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0b9e243d5211d809d600326d4c8f7918795a868d1acf0b91d9ee26f96d68d44a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"4e5c1277a0285514fbe13ffd0e118b84890287d84121f73af4e99275508596ce","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x242124721260\u003e: \n failed to create NodePool np-zb9djq: failed starting nodepool creation \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\" in resourcegroup=\"rg-nightly-4-20-r55xk7zmjr9c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-r55xk7zmjr9c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zb9djq/nodePools/np-zb9djq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x242124721260\u003e: \n failed to create NodePool np-zb9djq: failed starting nodepool creation \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\" in resourcegroup=\"rg-nightly-4-20-r55xk7zmjr9c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-r55xk7zmjr9c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zb9djq/nodePools/np-zb9djq\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"0c2ff36c9fa9d1c2814df9873f715f5d26c0e9c3b78280e32e1adf0c60c9ec65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"80a54d00d33387405efc62cd65294145e6838630c336a95f7908f0dc39f21afd","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x184038744400\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tzvzfh\" in resourcegroup=\"rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tzvzfh/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0x184038744400\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tzvzfh\" in resourcegroup=\"rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tzvzfh/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1a27268c2efba9efc20b39873ef90f27c7c0a87fd8db29be02e9ab410377bf8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"88bfecb223eed4d7950743d5a839fc1088854870e36ffeb70cfcf0e0e832d4ce","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x366855210620\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4c8kl482c2st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\",\n \"name\": \"37548aa4-a4e7-458d-b686-fb1201996386\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:15:28.253854505Z\",\n \"endTime\": \"2026-10-03T23:34:30.095088976Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: Creating: virtualmachine creating or updating\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x366855210620\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4c8kl482c2st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\",\n \"name\": \"37548aa4-a4e7-458d-b686-fb1201996386\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:15:28.253854505Z\",\n \"endTime\": \"2026-10-03T23:34:30.095088976Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: Creating: virtualmachine creating or updating\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"25160e6ee80916e150628d46923e30708eb79575039019904a641266cb9789ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"5c797d19c80475dc8cd016a2f12b5969e5f66674a64c327fd5b8176cbb909c6a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x31a06c9a5d00\u003e: \n failed to create NodePool np-dz68ws: failed waiting for nodepool=\"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\" in resourcegroup=\"rg-nightly-5-0-4t5m8n4czrvj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"name\": \"2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:35:55.566084896Z\",\n \"endTime\": \"2026-10-03T23:55:00.222357543Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: Creating: virtualmachine creating or updating\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x31a06c9a5d00\u003e: \n failed to create NodePool np-dz68ws: failed waiting for nodepool=\"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\" in resourcegroup=\"rg-nightly-5-0-4t5m8n4czrvj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"name\": \"2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:35:55.566084896Z\",\n \"endTime\": \"2026-10-03T23:55:00.222357543Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: Creating: virtualmachine creating or updating\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"311aed6905455f01a24ce18b915edf795a70b35896aaf94a63ce79bf1bbdb810","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"0c43de85fc2d6a7c071c55d1f96008747d85103a5e49f90377c2b68757383d44","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x36252fbe4930\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-548xlmq4vp84/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0x36252fbe4930\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-548xlmq4vp84/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"33e5084d82fc1f34d4dc1b1eee6e3646f5610b03a122a91f38ceae0075e4eca8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"4220fe7603791b9ddfb09840fb8cd4a59017f25f10351d3998d45dda9b1f0d93","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3102b57c8300\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-5xj4fb5fc6dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0x3102b57c8300\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-5xj4fb5fc6dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"63a18ee70b6dd3fadee56e8db1f5f3815a03c5ddd42622454beb047252f63f42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"390cd3c454160d58c3f66648e63a7907c2041ab6f7cec42614cfb83642a9075a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34ef387fcea0\u003e: \n failed to create NodePool np-vgspf2: failed waiting for nodepool=\"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\" in resourcegroup=\"rg-nightly-4-21-dsfs76js9854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"name\": \"78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:34:31.068400483Z\",\n \"endTime\": \"2026-10-03T23:53:40.358231747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: Creating: virtualmachine creating or updating\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34ef387fcea0\u003e: \n failed to create NodePool np-vgspf2: failed waiting for nodepool=\"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\" in resourcegroup=\"rg-nightly-4-21-dsfs76js9854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"name\": \"78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:34:31.068400483Z\",\n \"endTime\": \"2026-10-03T23:53:40.358231747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: Creating: virtualmachine creating or updating\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8577416f451a2ed36ed114a6b22811e1bb91aa89f9fd3decda2a04485274719f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"e9efc6e08915eb27cce0e9b2aee48e6761e029a06f9f351f49af927ae38ed282","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4pbb46\" for cluster \"cluster-nightly-5-1-4pbb46\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1fb215239360\u003e: \n failed to create NodePool np-4pbb46: failed waiting for nodepool=\"np-4pbb46\" for cluster \"cluster-nightly-5-1-4pbb46\" in resourcegroup=\"rg-nightly-5-1-4rftrv2tqf7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\",\n \"name\": \"e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:37:12.955436587Z\",\n \"endTime\": \"2026-10-03T23:56:20.22170931Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-4s8vt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-rvt6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-4s8vt: Creating: virtualmachine creating or updating\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-rvt6q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4pbb46\" for cluster \"cluster-nightly-5-1-4pbb46\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1fb215239360\u003e: \n failed to create NodePool np-4pbb46: failed waiting for nodepool=\"np-4pbb46\" for cluster \"cluster-nightly-5-1-4pbb46\" in resourcegroup=\"rg-nightly-5-1-4rftrv2tqf7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\",\n \"name\": \"e74d3e2c-deeb-4d38-9a72-e8d6c34c576d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:37:12.955436587Z\",\n \"endTime\": \"2026-10-03T23:56:20.22170931Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-4s8vt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-rvt6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-4s8vt: Creating: virtualmachine creating or updating\\nMachine d6a2h5t1r0s8k0n-np-4pbb46-vntxw-rvt6q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"99de6c78e3de7de9d60851b5d6fbcd83dfbfb01b29445379aed449ddc179129f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"9a1223f045ad2c43a2d2b9f5c9d15450c0250574d5337dd1a988d4371b014517","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x2f86d9c91d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-4kkvfds7xktp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0x2f86d9c91d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-4kkvfds7xktp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9e96f03b2056f80dcb253c5f798b3bf67b5e5d5b6323fdbc0266869acd8b507f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"e723e28bcd7e435dcc3421287fec024c49aa82e18f5dc42262402d55af5c17bd","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x145ed3760d50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-cxtlbfhqtdd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0x145ed3760d50\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-cxtlbfhqtdd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b946f60fd0e2b4c95b90842c71b4fb94c7565abfd4452b6a8c72d8cdd7d0bdca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"2b24f7ee1c45683523e52ab6a6fd28bf4368385c2a3884bc30ecbb5e5516b086","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x104897805d20\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r8db5d4ldfmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"name\": \"12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:41:40.611562032Z\",\n \"endTime\": \"2026-10-04T00:00:46.799966407Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: Creating: virtualmachine creating or updating\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x104897805d20\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r8db5d4ldfmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"name\": \"12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:41:40.611562032Z\",\n \"endTime\": \"2026-10-04T00:00:46.799966407Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: Creating: virtualmachine creating or updating\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management"}},{"row_id":"c8d4cbad50573632950a267892092074d3b0e567ad2d7352a039303c74fc38d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"1c86c022cf7d0c7808be0cd45d82052e35c88c4c35fb78fb4a2b9cf8b10f84d2","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x336e50b8a360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-njsbz992h4xx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0x336e50b8a360\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-njsbz992h4xx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d10b5b185889c09cae049e8dbd76615e7fac1711d9d0a2a267a45afb4178e88a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","failed_at":"e2e","signature_id":"f9761523403367d45961176e4cd187bb59f7a5f7a10bfd1b755e1c02f0042a38","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-hz7c47\" for cluster \"cluster-nightly-4-23-hz7c47\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33bfacf01ac0\u003e: \n failed to create NodePool np-hz7c47: failed waiting for nodepool=\"np-hz7c47\" for cluster \"cluster-nightly-4-23-hz7c47\" in resourcegroup=\"rg-nightly-4-23-zskzh2wkzsf8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ea1a45b2-8ea6-4372-9874-0ab41e3da185\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ea1a45b2-8ea6-4372-9874-0ab41e3da185\",\n \"name\": \"ea1a45b2-8ea6-4372-9874-0ab41e3da185\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:36:57.511630158Z\",\n \"endTime\": \"2026-10-03T23:56:00.189645328Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-hmv9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-qxrgm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-hmv9g: Creating: virtualmachine creating or updating\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-qxrgm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-hz7c47\" for cluster \"cluster-nightly-4-23-hz7c47\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x33bfacf01ac0\u003e: \n failed to create NodePool np-hz7c47: failed waiting for nodepool=\"np-hz7c47\" for cluster \"cluster-nightly-4-23-hz7c47\" in resourcegroup=\"rg-nightly-4-23-zskzh2wkzsf8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ea1a45b2-8ea6-4372-9874-0ab41e3da185\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ea1a45b2-8ea6-4372-9874-0ab41e3da185\",\n \"name\": \"ea1a45b2-8ea6-4372-9874-0ab41e3da185\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T23:36:57.511630158Z\",\n \"endTime\": \"2026-10-03T23:56:00.189645328Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-hmv9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-qxrgm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-hmv9g: Creating: virtualmachine creating or updating\\nMachine p8n6w8u1n2s0m9t-np-hz7c47-xs76h-qxrgm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":14,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106215444435177472","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T02:51:08Z","started_at":"2026-10-03T02:51:08Z","completed_at":"2026-10-03T03:55:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T04:04:04.941200231Z","timing_metadata_checked_at":"2026-10-03T04:04:04.941200231Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-03T04:04:04.945480217Z","region_metadata_checked_at":"2026-10-03T05:07:22.822875223Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"05519656a6426c31fdaeeeffe3f1eba7fd19497f6c622f9b4fc2310fd9ee2272","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106215444435177472","occurred_at":"2026-10-03T02:51:08Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T02:00:44Z","started_at":"2026-10-03T02:00:44Z","completed_at":"2026-10-03T04:45:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T04:56:48.069203567Z","timing_metadata_checked_at":"2026-10-03T04:56:48.069203567Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T04:56:48.072716054Z","region_metadata_checked_at":"2026-10-03T04:56:48.072716054Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"1e82b028368cdac827fbf6dec9f74e79fdf0f46fc0a15e33fa8725c647c8b6f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"2c2caa46b0c9a860c8536cb1d0cfe4dc435fe9c87395f2009a2b50d36e765530","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1634a92060e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-5tmnm5zpfp6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2872059a-968a-400a-8256-03392979308f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2872059a-968a-400a-8256-03392979308f\",\n \"name\": \"2872059a-968a-400a-8256-03392979308f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:59:17.985754909Z\",\n \"endTime\": \"2026-10-03T03:18:20.080261329Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-x5mt5-q58ts: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-x5mt5-q58ts: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1634a92060e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-5tmnm5zpfp6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2872059a-968a-400a-8256-03392979308f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2872059a-968a-400a-8256-03392979308f\",\n \"name\": \"2872059a-968a-400a-8256-03392979308f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:59:17.985754909Z\",\n \"endTime\": \"2026-10-03T03:18:20.080261329Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-x5mt5-q58ts: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-x5mt5-q58ts: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3e28209d0365c5b58e497a8180b38441bb85bcf23cd1705add37bbf1a0158370","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"6ed585f3f050ad8b54d850a6ae0fdd9b4909a19d9c493dbc159c6d4911096dd3","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1c6f23908030\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ddfqq694gxhd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bea67035-4bd4-49aa-97d8-eb0a71fb82dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bea67035-4bd4-49aa-97d8-eb0a71fb82dc\",\n \"name\": \"bea67035-4bd4-49aa-97d8-eb0a71fb82dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:59:14.55416176Z\",\n \"endTime\": \"2026-10-03T03:18:19.785989613Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-7d6m6-8kqhg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-7d6m6-ptnv4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-7d6m6-8kqhg: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-7d6m6-ptnv4: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1c6f23908030\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ddfqq694gxhd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bea67035-4bd4-49aa-97d8-eb0a71fb82dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bea67035-4bd4-49aa-97d8-eb0a71fb82dc\",\n \"name\": \"bea67035-4bd4-49aa-97d8-eb0a71fb82dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:59:14.55416176Z\",\n \"endTime\": \"2026-10-03T03:18:19.785989613Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-7d6m6-8kqhg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-7d6m6-ptnv4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-7d6m6-8kqhg: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-7d6m6-ptnv4: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4e78cd9ac7b4f2b18f4b62e1706dd30bccb5e1ebd552ab8af8bec65fc5705015","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"e9fa9ae8e858a7b5355cb1b43d0d6779abb4d6d970a6d15f0440c7b03e009e78","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wgvn44\" for cluster \"cluster-candidate-5-1-wgvn44\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x582bdd8e0c0\u003e: \n failed to create NodePool np-wgvn44: failed waiting for nodepool=\"np-wgvn44\" for cluster \"cluster-candidate-5-1-wgvn44\" in resourcegroup=\"rg-candidate-5-1-g5g8pnhdz6gx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b96711e-b171-48d1-87d7-1f055025bc87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b96711e-b171-48d1-87d7-1f055025bc87\",\n \"name\": \"7b96711e-b171-48d1-87d7-1f055025bc87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:24:01.084946017Z\",\n \"endTime\": \"2026-10-03T03:43:03.38976371Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-5mgvm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-vkmcn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-5mgvm: Creating: virtualmachine creating or updating\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-vkmcn: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wgvn44\" for cluster \"cluster-candidate-5-1-wgvn44\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x582bdd8e0c0\u003e: \n failed to create NodePool np-wgvn44: failed waiting for nodepool=\"np-wgvn44\" for cluster \"cluster-candidate-5-1-wgvn44\" in resourcegroup=\"rg-candidate-5-1-g5g8pnhdz6gx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b96711e-b171-48d1-87d7-1f055025bc87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7b96711e-b171-48d1-87d7-1f055025bc87\",\n \"name\": \"7b96711e-b171-48d1-87d7-1f055025bc87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:24:01.084946017Z\",\n \"endTime\": \"2026-10-03T03:43:03.38976371Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-5mgvm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-vkmcn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-5mgvm: Creating: virtualmachine creating or updating\\nMachine j6u5b3c2z9q1l8v-np-wgvn44-b9dp7-vkmcn: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"533cc6932efa5876a88364efb445e7b106967255edac7ab9cc37002d995d4219","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"9528ad897be3c780f7f61e03b1ee8a6fe0b00f376a6ff34c98cd8ee75346769c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-994gtb\" for cluster \"cluster-candidate-5-0-994gtb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b59f6a200a0\u003e: \n failed to create NodePool np-994gtb: failed waiting for nodepool=\"np-994gtb\" for cluster \"cluster-candidate-5-0-994gtb\" in resourcegroup=\"rg-candidate-5-0-dpqs94628k6g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f3ae09fa-a5f4-44d2-9f00-68063627145c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f3ae09fa-a5f4-44d2-9f00-68063627145c\",\n \"name\": \"f3ae09fa-a5f4-44d2-9f00-68063627145c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:48.672584866Z\",\n \"endTime\": \"2026-10-03T03:41:53.397092955Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-9xspx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-v8vvg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-9xspx: Creating: virtualmachine creating or updating\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-v8vvg: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-994gtb\" for cluster \"cluster-candidate-5-0-994gtb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3b59f6a200a0\u003e: \n failed to create NodePool np-994gtb: failed waiting for nodepool=\"np-994gtb\" for cluster \"cluster-candidate-5-0-994gtb\" in resourcegroup=\"rg-candidate-5-0-dpqs94628k6g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f3ae09fa-a5f4-44d2-9f00-68063627145c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f3ae09fa-a5f4-44d2-9f00-68063627145c\",\n \"name\": \"f3ae09fa-a5f4-44d2-9f00-68063627145c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:48.672584866Z\",\n \"endTime\": \"2026-10-03T03:41:53.397092955Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-9xspx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-v8vvg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-9xspx: Creating: virtualmachine creating or updating\\nMachine w0a5p3m7x2i2z6g-np-994gtb-827pv-v8vvg: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6f35cced70610dce46b749d6d7e1a72365d3fc153d4f4328bf225b9a12c8c996","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"fd808e83579b395ae5abd54c9de0aa37f2d5bc4ae6a53ba2e695ba0fa5c2dbc8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zsxxnl\" for cluster \"cluster-candidate-4-22-zsxxnl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x8479408b140\u003e: \n failed to create NodePool np-zsxxnl: failed waiting for nodepool=\"np-zsxxnl\" for cluster \"cluster-candidate-4-22-zsxxnl\" in resourcegroup=\"rg-candidate-4-22-7822rkz4qt8d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fd0879a-ebbb-4fea-b214-ece8494edab0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fd0879a-ebbb-4fea-b214-ece8494edab0\",\n \"name\": \"9fd0879a-ebbb-4fea-b214-ece8494edab0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:25.37619508Z\",\n \"endTime\": \"2026-10-03T03:41:30.020299253Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-6hmr2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-b6wrl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-6hmr2: Creating: virtualmachine creating or updating\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-b6wrl: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zsxxnl\" for cluster \"cluster-candidate-4-22-zsxxnl\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x8479408b140\u003e: \n failed to create NodePool np-zsxxnl: failed waiting for nodepool=\"np-zsxxnl\" for cluster \"cluster-candidate-4-22-zsxxnl\" in resourcegroup=\"rg-candidate-4-22-7822rkz4qt8d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fd0879a-ebbb-4fea-b214-ece8494edab0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9fd0879a-ebbb-4fea-b214-ece8494edab0\",\n \"name\": \"9fd0879a-ebbb-4fea-b214-ece8494edab0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:25.37619508Z\",\n \"endTime\": \"2026-10-03T03:41:30.020299253Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-6hmr2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-b6wrl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-6hmr2: Creating: virtualmachine creating or updating\\nMachine r5g9o6x4t5g2e7e-np-zsxxnl-wkz99-b6wrl: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"812731e7ce1217c0a8cf1d47e29afbce21140f7de2697c517ff08150e4fc7e1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"882fba6cad22093956999ee2384f3228d45b1777a330537ec945b37008289a0a","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x933c5f12a60\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lbkmsp45kpxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\",\n \"name\": \"efe85861-3de4-4231-94f4-c87043439307\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:54:52.260057343Z\",\n \"endTime\": \"2026-10-03T04:13:53.608229594Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: Creating: virtualmachine creating or updating\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0x933c5f12a60\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lbkmsp45kpxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\",\n \"name\": \"efe85861-3de4-4231-94f4-c87043439307\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:54:52.260057343Z\",\n \"endTime\": \"2026-10-03T04:13:53.608229594Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: Creating: virtualmachine creating or updating\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management"}},{"row_id":"8167bec06508c0a0cb844a4e4c1a98c880b64dfca1cb775ca53050adf2c6443d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"39751860e6d26d74a5f620ec93a78ea99d0e96ecd0ed7587dea64af7ba46a985","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xe7412ec40a8\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-j2962r94fd6x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"name\": \"4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:00:19.581988952Z\",\n \"endTime\": \"2026-10-03T03:19:19.770602537Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xe7412ec40a8\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-j2962r94fd6x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"name\": \"4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:00:19.581988952Z\",\n \"endTime\": \"2026-10-03T03:19:19.770602537Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"986d7547b214789d5aea01f33a08eddf88dfb49f80123dcaa2ff9b05368cefa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"8ed18e43fabdf08315daac2d8e02c94f2c8a172b70d38853c874dbc955f5d982","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x21c259b32dc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-2sgbstf7nlxj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3898db04-618a-4491-92e8-a44483e69093\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3898db04-618a-4491-92e8-a44483e69093\",\n \"name\": \"3898db04-618a-4491-92e8-a44483e69093\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:58:32.753298506Z\",\n \"endTime\": \"2026-10-03T03:17:39.788365749Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-hbt2g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-zpxs2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-hbt2g: Creating: virtualmachine creating or updating\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-zpxs2: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x21c259b32dc0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-2sgbstf7nlxj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3898db04-618a-4491-92e8-a44483e69093\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3898db04-618a-4491-92e8-a44483e69093\",\n \"name\": \"3898db04-618a-4491-92e8-a44483e69093\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T02:58:32.753298506Z\",\n \"endTime\": \"2026-10-03T03:17:39.788365749Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-hbt2g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-zpxs2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-hbt2g: Creating: virtualmachine creating or updating\\nMachine m5r0r0n9t0l1p2d-np-1-dr752-zpxs2: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a64bfbf378aa9bc2d91396a3c062810db32dc76603cadbc28cb269b1b69e317d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"8dc98ddc487d792e30d36e88cd34b10e8b904d76ef343c1f573a64f712f3b220","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x15f1e13c9ee0\u003e: \n failed to create NodePool np-6tqvsp: failed waiting for nodepool=\"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\" in resourcegroup=\"rg-candidate-4-21-qxp54ksqdkvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"name\": \"cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:23:37.870437581Z\",\n \"endTime\": \"2026-10-03T03:42:43.397028044Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: Creating: virtualmachine creating or updating\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x15f1e13c9ee0\u003e: \n failed to create NodePool np-6tqvsp: failed waiting for nodepool=\"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\" in resourcegroup=\"rg-candidate-4-21-qxp54ksqdkvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"name\": \"cd90e427-e90a-4449-9d9c-98919a3e2323\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:23:37.870437581Z\",\n \"endTime\": \"2026-10-03T03:42:43.397028044Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: Creating: virtualmachine creating or updating\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cd8ada755a7b03b78d2ecbcd9dbb6d9a148fa6a6a8f57a8233b64c5a9f879eac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"e5c35e2b9b861c774ee1789f184fbf1cb2f5f308d8dc05c180346b85e646fe05","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vlg9bq\" for cluster \"cluster-candidate-4-23-vlg9bq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e6e62f6cda0\u003e: \n failed to create NodePool np-vlg9bq: failed waiting for nodepool=\"np-vlg9bq\" for cluster \"cluster-candidate-4-23-vlg9bq\" in resourcegroup=\"rg-candidate-4-23-dfclsfklgxtx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98169a27-755e-48b1-800f-32bc5e94d96c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98169a27-755e-48b1-800f-32bc5e94d96c\",\n \"name\": \"98169a27-755e-48b1-800f-32bc5e94d96c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:35.181930151Z\",\n \"endTime\": \"2026-10-03T03:41:39.859849458Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-mqggg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-zfwnx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-mqggg: Creating: virtualmachine creating or updating\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-zfwnx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vlg9bq\" for cluster \"cluster-candidate-4-23-vlg9bq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x1e6e62f6cda0\u003e: \n failed to create NodePool np-vlg9bq: failed waiting for nodepool=\"np-vlg9bq\" for cluster \"cluster-candidate-4-23-vlg9bq\" in resourcegroup=\"rg-candidate-4-23-dfclsfklgxtx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98169a27-755e-48b1-800f-32bc5e94d96c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98169a27-755e-48b1-800f-32bc5e94d96c\",\n \"name\": \"98169a27-755e-48b1-800f-32bc5e94d96c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:22:35.181930151Z\",\n \"endTime\": \"2026-10-03T03:41:39.859849458Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-mqggg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-zfwnx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-mqggg: Creating: virtualmachine creating or updating\\nMachine g4t5b9n2y6n5g3x-np-vlg9bq-nplgj-zfwnx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e626ee7d005666c5d5f396c38b3aef1b15642cd86798c690de5419fc7fa8f199","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","failed_at":"e2e","signature_id":"2c9f39e0ab026a6227defad21130e9e9d394ff6a055a7d1fa925b7fc0262141f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7k8q7d\" for cluster \"cluster-candidate-4-20-7k8q7d\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x342eb43e6c40\u003e: \n failed to create NodePool np-7k8q7d: failed waiting for nodepool=\"np-7k8q7d\" for cluster \"cluster-candidate-4-20-7k8q7d\" in resourcegroup=\"rg-candidate-4-20-vn2gzb9bgx68\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f5da487-7370-42cc-8931-634a89f3149c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f5da487-7370-42cc-8931-634a89f3149c\",\n \"name\": \"8f5da487-7370-42cc-8931-634a89f3149c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:18:20.263677419Z\",\n \"endTime\": \"2026-10-03T03:37:30.013101747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-rlpsx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-v5gzq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-rlpsx: Creating: virtualmachine creating or updating\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-v5gzq: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7k8q7d\" for cluster \"cluster-candidate-4-20-7k8q7d\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x342eb43e6c40\u003e: \n failed to create NodePool np-7k8q7d: failed waiting for nodepool=\"np-7k8q7d\" for cluster \"cluster-candidate-4-20-7k8q7d\" in resourcegroup=\"rg-candidate-4-20-vn2gzb9bgx68\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f5da487-7370-42cc-8931-634a89f3149c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f5da487-7370-42cc-8931-634a89f3149c\",\n \"name\": \"8f5da487-7370-42cc-8931-634a89f3149c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T03:18:20.263677419Z\",\n \"endTime\": \"2026-10-03T03:37:30.013101747Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-rlpsx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-v5gzq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-rlpsx: Creating: virtualmachine creating or updating\\nMachine n2s3g3y6j4i7c5t-np-7k8q7d-vrbbm-v5gzq: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-03T00:25:08Z","started_at":"2026-10-03T00:25:08Z","completed_at":"2026-10-03T02:43:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T03:00:49.285108674Z","timing_metadata_checked_at":"2026-10-03T03:00:49.285108674Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T03:00:49.288123164Z","region_metadata_checked_at":"2026-10-03T03:00:49.288123164Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"32dd870e25444edc34f0a54c4f5de17f756f32c1abedbbc2dff3c7f667d5bc09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"d6f5de42eefb7152643e950681559e711013d084e28baf1d882e6f029c0e5511","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xca0cd836f80\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"name\": \"4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:51:49.926221366Z\",\n \"endTime\": \"2026-10-03T02:10:56.26372719Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: Creating: virtualmachine creating or updating\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xca0cd836f80\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"name\": \"4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:51:49.926221366Z\",\n \"endTime\": \"2026-10-03T02:10:56.26372719Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: Creating: virtualmachine creating or updating\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management"}},{"row_id":"46b26e75380dd03099914ca7e74dae6badab952d4d44aae6a15e8e2881d52082","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"475946b6879259ac2394beb241c800f706ea9c7b18c2b17cfb61377bbdc2cf11","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1363e14cc018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-jtmvgv9m6829\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"name\": \"31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:01:05.764058109Z\",\n \"endTime\": \"2026-10-03T01:20:12.852653102Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x1363e14cc018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-jtmvgv9m6829\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"name\": \"31894849-27e9-4053-a67e-9e3babf2ad8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:01:05.764058109Z\",\n \"endTime\": \"2026-10-03T01:20:12.852653102Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"55886caa49c7a303baada857ec16c42b3ce5bb633b0bed38f3bd2d35c9cb2f49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"2270ab4d2942cd093aff353fd34037a69c93e335ec7f1a88dfa214a3f4004534","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7m65d4\" for cluster \"cluster-candidate-4-21-7m65d4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xdd74a91040\u003e: \n failed to create NodePool np-7m65d4: failed waiting for nodepool=\"np-7m65d4\" for cluster \"cluster-candidate-4-21-7m65d4\" in resourcegroup=\"rg-candidate-4-21-cdkldcpn7c4w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f45b9d58-6bb8-408f-943e-164ec6193300\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f45b9d58-6bb8-408f-943e-164ec6193300\",\n \"name\": \"f45b9d58-6bb8-408f-943e-164ec6193300\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:23:06.848197915Z\",\n \"endTime\": \"2026-10-03T01:42:11.810063991Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-7jtvb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-8fd6d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-7jtvb: Creating: virtualmachine creating or updating\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-8fd6d: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7m65d4\" for cluster \"cluster-candidate-4-21-7m65d4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xdd74a91040\u003e: \n failed to create NodePool np-7m65d4: failed waiting for nodepool=\"np-7m65d4\" for cluster \"cluster-candidate-4-21-7m65d4\" in resourcegroup=\"rg-candidate-4-21-cdkldcpn7c4w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f45b9d58-6bb8-408f-943e-164ec6193300\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f45b9d58-6bb8-408f-943e-164ec6193300\",\n \"name\": \"f45b9d58-6bb8-408f-943e-164ec6193300\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:23:06.848197915Z\",\n \"endTime\": \"2026-10-03T01:42:11.810063991Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-7jtvb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-8fd6d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-7jtvb: Creating: virtualmachine creating or updating\\nMachine z8t8u8s2s4m5t9g-np-7m65d4-6msht-8fd6d: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"713dd46977abda13d0ec8b75a9ad33bbcccef6aff09092181e72e6d01b9456d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"8f5972671bc46aef70b454f5d37eb1f821ffe511b2716373992fdcfdd6e793dd","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x356eae117080\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-2xqzzh8h7jx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2692539-c900-4d9b-9952-7a76ad7fc6a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2692539-c900-4d9b-9952-7a76ad7fc6a0\",\n \"name\": \"c2692539-c900-4d9b-9952-7a76ad7fc6a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:56:20.964200965Z\",\n \"endTime\": \"2026-10-03T01:15:22.786636188Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-kbv8l-5qvk6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-kbv8l-5qvk6: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x356eae117080\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-2xqzzh8h7jx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2692539-c900-4d9b-9952-7a76ad7fc6a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c2692539-c900-4d9b-9952-7a76ad7fc6a0\",\n \"name\": \"c2692539-c900-4d9b-9952-7a76ad7fc6a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:56:20.964200965Z\",\n \"endTime\": \"2026-10-03T01:15:22.786636188Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-kbv8l-5qvk6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-kbv8l-5qvk6: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7970d9eef7bcdbc5fdd8d98ccb5de27654e4a45317dcb366505f419f46508f5e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"2a69c285def7ab9febd458730c74909c9a16be32934cbdbd5ca44844262b1eb0","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x36891cc75140\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-rw4pgrx2qsqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a719f074-9cbb-4f60-8f74-9acd323de195\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a719f074-9cbb-4f60-8f74-9acd323de195\",\n \"name\": \"a719f074-9cbb-4f60-8f74-9acd323de195\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:58:50.620270926Z\",\n \"endTime\": \"2026-10-03T01:17:53.07846241Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-nmjht: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-qnrnm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-nmjht: Creating: virtualmachine creating or updating\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-qnrnm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x36891cc75140\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-rw4pgrx2qsqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a719f074-9cbb-4f60-8f74-9acd323de195\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a719f074-9cbb-4f60-8f74-9acd323de195\",\n \"name\": \"a719f074-9cbb-4f60-8f74-9acd323de195\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:58:50.620270926Z\",\n \"endTime\": \"2026-10-03T01:17:53.07846241Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-nmjht: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-qnrnm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-nmjht: Creating: virtualmachine creating or updating\\nMachine y4c5l7p2o7h6i3y-np-1-7slbg-qnrnm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"89d87ec14c3e09c46fef3fce7a4099ca7d20f514ce909b2e541481cf32e92feb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"06c296f07902755728beb3838eb1ec4ac170f38778b421b87c5fc204e64d1fe4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7bgsbj\" for cluster \"cluster-candidate-5-1-7bgsbj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x355ed6679960\u003e: \n failed to create NodePool np-7bgsbj: failed waiting for nodepool=\"np-7bgsbj\" for cluster \"cluster-candidate-5-1-7bgsbj\" in resourcegroup=\"rg-candidate-5-1-twzxzjn6gcp4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48ddacde-704d-413e-818f-fb753e5b6f29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48ddacde-704d-413e-818f-fb753e5b6f29\",\n \"name\": \"48ddacde-704d-413e-818f-fb753e5b6f29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:21:40.063888823Z\",\n \"endTime\": \"2026-10-03T01:40:41.803400223Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-b2m7f: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-ssxpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-b2m7f: Creating: virtualmachine creating or updating\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-ssxpc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7bgsbj\" for cluster \"cluster-candidate-5-1-7bgsbj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x355ed6679960\u003e: \n failed to create NodePool np-7bgsbj: failed waiting for nodepool=\"np-7bgsbj\" for cluster \"cluster-candidate-5-1-7bgsbj\" in resourcegroup=\"rg-candidate-5-1-twzxzjn6gcp4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48ddacde-704d-413e-818f-fb753e5b6f29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/48ddacde-704d-413e-818f-fb753e5b6f29\",\n \"name\": \"48ddacde-704d-413e-818f-fb753e5b6f29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:21:40.063888823Z\",\n \"endTime\": \"2026-10-03T01:40:41.803400223Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-b2m7f: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-ssxpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-b2m7f: Creating: virtualmachine creating or updating\\nMachine h4l9i6c5a5m3n3t-np-7bgsbj-b5gdb-ssxpc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8b0ebde71eb2f691daf70a2bcd3cc4703f41ec171e90b55dfcf6dfefbcc160bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"ed4c47b0094dced5127989ca5fda7f6a4352c3e90ee689cf14e5def8764d9f1d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x382d24c28030\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-zvljfb77qpnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd6903c4-bc0c-4634-b456-f0053f4e62ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd6903c4-bc0c-4634-b456-f0053f4e62ce\",\n \"name\": \"fd6903c4-bc0c-4634-b456-f0053f4e62ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:59:54.046743948Z\",\n \"endTime\": \"2026-10-03T01:19:03.00985984Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-2hhfv-7r58z: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-2hhfv-fmwpb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-2hhfv-7r58z: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-2hhfv-fmwpb: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0x382d24c28030\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-zvljfb77qpnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd6903c4-bc0c-4634-b456-f0053f4e62ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fd6903c4-bc0c-4634-b456-f0053f4e62ce\",\n \"name\": \"fd6903c4-bc0c-4634-b456-f0053f4e62ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:59:54.046743948Z\",\n \"endTime\": \"2026-10-03T01:19:03.00985984Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-2hhfv-7r58z: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-2hhfv-fmwpb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-2hhfv-7r58z: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-2hhfv-fmwpb: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a90f150f6c71ea9e51ea7d96e09f6f2f22abc4c9d527371c73769d9f53c0f0a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"173a7e9edaf6f61e13589f056617df6ddb63fa543ce96dbe851cba6372717a02","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-mpbw9s\" for cluster \"cluster-candidate-5-0-mpbw9s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xea20e6b7700\u003e: \n failed to create NodePool np-mpbw9s: failed waiting for nodepool=\"np-mpbw9s\" for cluster \"cluster-candidate-5-0-mpbw9s\" in resourcegroup=\"rg-candidate-5-0-6l9pw288lpdv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f8bd06-343a-4b31-8a4d-21de9398ca35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f8bd06-343a-4b31-8a4d-21de9398ca35\",\n \"name\": \"49f8bd06-343a-4b31-8a4d-21de9398ca35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:23:58.4098212Z\",\n \"endTime\": \"2026-10-03T01:43:01.804835963Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-sx8g2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-xvdjz: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-sx8g2: Creating: virtualmachine creating or updating\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-xvdjz: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-mpbw9s\" for cluster \"cluster-candidate-5-0-mpbw9s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xea20e6b7700\u003e: \n failed to create NodePool np-mpbw9s: failed waiting for nodepool=\"np-mpbw9s\" for cluster \"cluster-candidate-5-0-mpbw9s\" in resourcegroup=\"rg-candidate-5-0-6l9pw288lpdv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f8bd06-343a-4b31-8a4d-21de9398ca35\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/49f8bd06-343a-4b31-8a4d-21de9398ca35\",\n \"name\": \"49f8bd06-343a-4b31-8a4d-21de9398ca35\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:23:58.4098212Z\",\n \"endTime\": \"2026-10-03T01:43:01.804835963Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-sx8g2: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-xvdjz: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-sx8g2: Creating: virtualmachine creating or updating\\nMachine k9g6t4i6h6q8k7w-np-mpbw9s-tmszd-xvdjz: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"aa6e65fd1b264e949cc75090b7fc457a827a9e08168675f5958a989ac9458eb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"34a4dddd4ac406094ca0f1a4bf7eb674580835c68e05a723aea1b94befa47471","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-55sh69\" for cluster \"cluster-candidate-4-23-55sh69\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d83ca910b00\u003e: \n failed to create NodePool np-55sh69: failed waiting for nodepool=\"np-55sh69\" for cluster \"cluster-candidate-4-23-55sh69\" in resourcegroup=\"rg-candidate-4-23-b2jh9rr7848q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/857aaaff-55f2-4507-bff3-f5f99ad82196\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/857aaaff-55f2-4507-bff3-f5f99ad82196\",\n \"name\": \"857aaaff-55f2-4507-bff3-f5f99ad82196\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:21:00.037991541Z\",\n \"endTime\": \"2026-10-03T01:40:02.219519171Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-qt7pp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-xbwxp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-qt7pp: Creating: virtualmachine creating or updating\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-xbwxp: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-55sh69\" for cluster \"cluster-candidate-4-23-55sh69\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x2d83ca910b00\u003e: \n failed to create NodePool np-55sh69: failed waiting for nodepool=\"np-55sh69\" for cluster \"cluster-candidate-4-23-55sh69\" in resourcegroup=\"rg-candidate-4-23-b2jh9rr7848q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/857aaaff-55f2-4507-bff3-f5f99ad82196\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/857aaaff-55f2-4507-bff3-f5f99ad82196\",\n \"name\": \"857aaaff-55f2-4507-bff3-f5f99ad82196\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:21:00.037991541Z\",\n \"endTime\": \"2026-10-03T01:40:02.219519171Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-qt7pp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-xbwxp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-qt7pp: Creating: virtualmachine creating or updating\\nMachine z7q7z3g1g0w5l4w-np-55sh69-kmbcq-xbwxp: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b6acb3809f5fabcdab967ed5a2340883e9c3f3707e6dd000e57e721aff58222d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"38fdd980bf52332969e2e4f977878b2b3caffd7b67413af3dedd10df203bfc1a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xcs2zx\" for cluster \"cluster-candidate-4-20-xcs2zx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bb65f4dc3e0\u003e: \n failed to create NodePool np-xcs2zx: failed waiting for nodepool=\"np-xcs2zx\" for cluster \"cluster-candidate-4-20-xcs2zx\" in resourcegroup=\"rg-candidate-4-20-9tm8m64jwh6q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5e3d740-6442-4b8e-9013-9867a8b30bc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5e3d740-6442-4b8e-9013-9867a8b30bc0\",\n \"name\": \"b5e3d740-6442-4b8e-9013-9867a8b30bc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:20:28.820955232Z\",\n \"endTime\": \"2026-10-03T01:39:31.794768159Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-42gm5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-zmvl5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-42gm5: Creating: virtualmachine creating or updating\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-zmvl5: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xcs2zx\" for cluster \"cluster-candidate-4-20-xcs2zx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x3bb65f4dc3e0\u003e: \n failed to create NodePool np-xcs2zx: failed waiting for nodepool=\"np-xcs2zx\" for cluster \"cluster-candidate-4-20-xcs2zx\" in resourcegroup=\"rg-candidate-4-20-9tm8m64jwh6q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5e3d740-6442-4b8e-9013-9867a8b30bc0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b5e3d740-6442-4b8e-9013-9867a8b30bc0\",\n \"name\": \"b5e3d740-6442-4b8e-9013-9867a8b30bc0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:20:28.820955232Z\",\n \"endTime\": \"2026-10-03T01:39:31.794768159Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-42gm5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-zmvl5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-42gm5: Creating: virtualmachine creating or updating\\nMachine b5o5a6z5q4p5e0b-np-xcs2zx-hlrdg-zmvl5: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c414687d08f84faa025d931b3e0814969e9ebb6afca24f6d5dab4384b1ca5a07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"399671fb721e085227d1bcef5acf1cf7744be28e89d36f981b42cc7c1ec8a4d7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-md77w4\" for cluster \"cluster-candidate-4-22-md77w4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34d9aee24b60\u003e: \n failed to create NodePool np-md77w4: failed waiting for nodepool=\"np-md77w4\" for cluster \"cluster-candidate-4-22-md77w4\" in resourcegroup=\"rg-candidate-4-22-qt94z5jzjgsz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7b70034-c32d-4689-8c16-d6b19ed89cc8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7b70034-c32d-4689-8c16-d6b19ed89cc8\",\n \"name\": \"c7b70034-c32d-4689-8c16-d6b19ed89cc8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:25:32.783100892Z\",\n \"endTime\": \"2026-10-03T01:44:41.785015844Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-488zq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-f2khf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-488zq: Creating: virtualmachine creating or updating\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-f2khf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-md77w4\" for cluster \"cluster-candidate-4-22-md77w4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0x34d9aee24b60\u003e: \n failed to create NodePool np-md77w4: failed waiting for nodepool=\"np-md77w4\" for cluster \"cluster-candidate-4-22-md77w4\" in resourcegroup=\"rg-candidate-4-22-qt94z5jzjgsz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7b70034-c32d-4689-8c16-d6b19ed89cc8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7b70034-c32d-4689-8c16-d6b19ed89cc8\",\n \"name\": \"c7b70034-c32d-4689-8c16-d6b19ed89cc8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:25:32.783100892Z\",\n \"endTime\": \"2026-10-03T01:44:41.785015844Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-488zq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-f2khf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-488zq: Creating: virtualmachine creating or updating\\nMachine m6p3c2b3z7i0g2t-np-md77w4-k4nbk-f2khf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f30cc3ab26e40e608530edfdf1bd27479dccbc04d61f493b4ffee7dc3136f009","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","failed_at":"e2e","signature_id":"d5c37afd72eeef3b1717f0ed626a3a3f20e451ad1fbcbdcf4ca6127c60afb433","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xbbf668ab320\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lqwbkg\" in resourcegroup=\"rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"name\": \"a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:58:23.770970582Z\",\n \"endTime\": \"2026-10-03T02:17:26.319658092Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xbbf668ab320\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lqwbkg\" in resourcegroup=\"rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"name\": \"a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T01:58:23.770970582Z\",\n \"endTime\": \"2026-10-03T02:17:26.319658092Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}