{"meta":{"date":"2026-10-27","timezone":"UTC","generated_at":"2026-10-28T23:57:30Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":6,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2115217181124857856","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T23:00:49Z","started_at":"2026-10-27T23:00:49Z","completed_at":"2026-10-28T01:24:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T01:33:49.861184485Z","timing_metadata_checked_at":"2026-10-28T01:33:49.861184485Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T01:33:49.864106477Z","region_metadata_checked_at":"2026-10-28T01:33:49.864106477Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d12ab05ef08eb3b266d20fcda230e2077630293667a94e04dc562a31ddcc16e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2115217181124857856","occurred_at":"2026-10-27T23:00:49Z","failed_at":"e2e","signature_id":"2081f477b523b7866b3f60de53b0fb6ab5d90668ea1f7f412ab3ed026d28073d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:168]: failed to verify HCP cluster \"cluster-nightly-4-21-htnxwk\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x28946bc08d20\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x28946b722320\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b722300\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b314d20\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x28946b60da40\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b60da20\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b518dc0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:168]: failed to verify HCP cluster \"cluster-nightly-4-21-htnxwk\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x28946bc08d20\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x28946b722320\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b722300\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b314d20\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x28946b60da40\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x28946b60da20\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x28946b518dc0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"96eb8165e1517c94f94344a079943853da38a18f87e198ac97e2ca5f84f1f2e3","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["96eb8165e1517c94f94344a079943853da38a18f87e198ac97e2ca5f84f1f2e3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2115190051670331392","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T21:13:01Z","started_at":"2026-10-27T21:13:01Z","completed_at":"2026-10-27T23:52:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-28T00:09:55.885026163Z","timing_metadata_checked_at":"2026-10-28T00:09:55.885026163Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-28T00:09:55.888139854Z","region_metadata_checked_at":"2026-10-28T00:09:55.888139854Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2115049607502237696","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T11:54:56Z","started_at":"2026-10-27T11:54:56Z","completed_at":"2026-10-27T14:32:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T14:43:29.033589366Z","timing_metadata_checked_at":"2026-10-27T14:43:29.033589366Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T14:43:29.036650655Z","region_metadata_checked_at":"2026-10-27T14:43:29.036650655Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T07:33:22Z","started_at":"2026-10-27T07:33:22Z","completed_at":"2026-10-27T10:03:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T10:12:08.889956806Z","timing_metadata_checked_at":"2026-10-27T10:12:08.889956806Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T10:12:08.893149594Z","region_metadata_checked_at":"2026-10-27T10:12:08.893149594Z"},"failed_at":["unknown"],"failed_test_count":12,"occurrences":[{"row_id":"00dd117c42b0dae3bcdcaa0b9d1e4e68657491f76d0f6f6202fc618cb6a1e3f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"d84676500a8d1f8f5aba72d56f9519c4ab3fe5ba3beacd4c258d168261c4440e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-b485rw\" for cluster \"cluster-candidate-4-23-b485rw\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1075b7e74980\u003e: \n    failed to create NodePool np-b485rw: failed waiting for nodepool=\"np-b485rw\" for cluster \"cluster-candidate-4-23-b485rw\" in resourcegroup=\"rg-candidate-4-23-khfknr4zp64h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f65d5700-0d56-4f82-b528-c8a03c20b091\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f65d5700-0d56-4f82-b528-c8a03c20b091\",\n      \"name\": \"f65d5700-0d56-4f82-b528-c8a03c20b091\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:33:58.455860138Z\",\n      \"endTime\": \"2026-10-27T08:52:58.299035711Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c0c5j0l8n8r8a3r-np-b485rw-67wmg-9gvgj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine c0c5j0l8n8r8a3r-np-b485rw-67wmg-bgdzh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c0c5j0l8n8r8a3r-np-b485rw-67wmg-9gvgj: Creating: virtualmachine creating or updating\\nMachine c0c5j0l8n8r8a3r-np-b485rw-67wmg-bgdzh: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f65d5700-0d56-4f82-b528-c8a03c20b091\n            correlation ID: 9b1ecdea-62ca-4b99-996f-b8c59602813a\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-b485rw\" for cluster \"cluster-candidate-4-23-b485rw\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1075b7e74980\u003e: \n    failed to create NodePool np-b485rw: failed waiting for nodepool=\"np-b485rw\" for cluster \"cluster-candidate-4-23-b485rw\" in resourcegroup=\"rg-candidate-4-23-khfknr4zp64h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f65d5700-0d56-4f82-b528-c8a03c20b091\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f65d5700-0d56-4f82-b528-c8a03c20b091\",\n      \"name\": \"f65d5700-0d56-4f82-b528-c8a03c20b091\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:33:58.455860138Z\",\n      \"endTime\": \"2026-10-27T08:52:58.299035711Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c0c5j0l8n8r8a3r-np-b485rw-67wmg-9gvgj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine c0c5j0l8n8r8a3r-np-b485rw-67wmg-bgdzh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c0c5j0l8n8r8a3r-np-b485rw-67wmg-9gvgj: Creating: virtualmachine creating or updating\\nMachine c0c5j0l8n8r8a3r-np-b485rw-67wmg-bgdzh: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f65d5700-0d56-4f82-b528-c8a03c20b091\n            correlation ID: 9b1ecdea-62ca-4b99-996f-b8c59602813a\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"03434b7af1394fb02a6655b2729addf3f759a2bb8f6407701bc930eb659de936","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"bdeacb1eac1daccd351ab2768ef041b978c2eb7deaf336006af84dd3f773eae0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-pwr5hm\" for cluster \"cluster-candidate-5-0-pwr5hm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a6192d956a0\u003e: \n    failed to create NodePool np-pwr5hm: failed waiting for nodepool=\"np-pwr5hm\" for cluster \"cluster-candidate-5-0-pwr5hm\" in resourcegroup=\"rg-candidate-5-0-7hv6h2jrxcx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c22a5a79-f84e-4aa7-9d3b-91a7d7558e28\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c22a5a79-f84e-4aa7-9d3b-91a7d7558e28\",\n      \"name\": \"c22a5a79-f84e-4aa7-9d3b-91a7d7558e28\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:36:44.781927018Z\",\n      \"endTime\": \"2026-10-27T08:55:48.199837161Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h7y2l9q7w3r4t6i-np-pwr5hm-f6g86-k4b6d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h7y2l9q7w3r4t6i-np-pwr5hm-f6g86-trw27: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h7y2l9q7w3r4t6i-np-pwr5hm-f6g86-k4b6d: Creating: virtualmachine creating or updating\\nMachine h7y2l9q7w3r4t6i-np-pwr5hm-f6g86-trw27: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c22a5a79-f84e-4aa7-9d3b-91a7d7558e28\n            correlation ID: fc347c43-9125-439c-9002-0dcad1e9a2a5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-pwr5hm\" for cluster \"cluster-candidate-5-0-pwr5hm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a6192d956a0\u003e: \n    failed to create NodePool np-pwr5hm: failed waiting for nodepool=\"np-pwr5hm\" for cluster \"cluster-candidate-5-0-pwr5hm\" in resourcegroup=\"rg-candidate-5-0-7hv6h2jrxcx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c22a5a79-f84e-4aa7-9d3b-91a7d7558e28\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c22a5a79-f84e-4aa7-9d3b-91a7d7558e28\",\n      \"name\": \"c22a5a79-f84e-4aa7-9d3b-91a7d7558e28\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:36:44.781927018Z\",\n      \"endTime\": \"2026-10-27T08:55:48.199837161Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h7y2l9q7w3r4t6i-np-pwr5hm-f6g86-k4b6d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h7y2l9q7w3r4t6i-np-pwr5hm-f6g86-trw27: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h7y2l9q7w3r4t6i-np-pwr5hm-f6g86-k4b6d: Creating: virtualmachine creating or updating\\nMachine h7y2l9q7w3r4t6i-np-pwr5hm-f6g86-trw27: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c22a5a79-f84e-4aa7-9d3b-91a7d7558e28\n            correlation ID: fc347c43-9125-439c-9002-0dcad1e9a2a5\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2b2ee1c9975f59475889729b54e75dab8b217675bd27cc36eb90f87c11d32629","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"aac52a12f1e11813e645caf10fcf25877853fed7ef788c7c5c5971b57aebcaa4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-ml8dz8\" for cluster \"cluster-candidate-4-20-ml8dz8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x25c240586440\u003e: \n    failed to create NodePool np-ml8dz8: failed waiting for nodepool=\"np-ml8dz8\" for cluster \"cluster-candidate-4-20-ml8dz8\" in resourcegroup=\"rg-candidate-4-20-jk94bqp4t7c4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca9d18db-ad83-4257-86ca-b626b3a1c04e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca9d18db-ad83-4257-86ca-b626b3a1c04e\",\n      \"name\": \"ca9d18db-ad83-4257-86ca-b626b3a1c04e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:32:49.74394684Z\",\n      \"endTime\": \"2026-10-27T08:51:58.323448782Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t2v7o2w6e3c2h7k-np-ml8dz8-lm8lp-bw7gg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine t2v7o2w6e3c2h7k-np-ml8dz8-lm8lp-hkpd5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t2v7o2w6e3c2h7k-np-ml8dz8-lm8lp-bw7gg: Creating: virtualmachine creating or updating\\nMachine t2v7o2w6e3c2h7k-np-ml8dz8-lm8lp-hkpd5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca9d18db-ad83-4257-86ca-b626b3a1c04e\n            correlation ID: b6d3f936-bcc1-4c84-b01a-da9310e58e24\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-ml8dz8\" for cluster \"cluster-candidate-4-20-ml8dz8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x25c240586440\u003e: \n    failed to create NodePool np-ml8dz8: failed waiting for nodepool=\"np-ml8dz8\" for cluster \"cluster-candidate-4-20-ml8dz8\" in resourcegroup=\"rg-candidate-4-20-jk94bqp4t7c4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca9d18db-ad83-4257-86ca-b626b3a1c04e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca9d18db-ad83-4257-86ca-b626b3a1c04e\",\n      \"name\": \"ca9d18db-ad83-4257-86ca-b626b3a1c04e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:32:49.74394684Z\",\n      \"endTime\": \"2026-10-27T08:51:58.323448782Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t2v7o2w6e3c2h7k-np-ml8dz8-lm8lp-bw7gg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine t2v7o2w6e3c2h7k-np-ml8dz8-lm8lp-hkpd5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t2v7o2w6e3c2h7k-np-ml8dz8-lm8lp-bw7gg: Creating: virtualmachine creating or updating\\nMachine t2v7o2w6e3c2h7k-np-ml8dz8-lm8lp-hkpd5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca9d18db-ad83-4257-86ca-b626b3a1c04e\n            correlation ID: b6d3f936-bcc1-4c84-b01a-da9310e58e24\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2df044e7e26281e151d916ae80bf365ac0b66bbd3207c43e14da0e7e99e45756","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"b9c3c5012ff77c3681e350e06c44d7277c66955d064a006d326eb01c4d066632","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3b12e2cba60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4j5vtdb84h8v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"name\": \"083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T09:13:51.032005174Z\",\n      \"endTime\": \"2026-10-27T09:32:53.54866754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: Creating: virtualmachine creating or updating\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n            correlation ID: 01ff1c80-ae7b-4cd9-aeb7-68197786fff9\n            client request ID: unavailable\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3b12e2cba60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4j5vtdb84h8v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"name\": \"083c3fe4-0c43-4b6a-9905-e8c1aa97e192\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T09:13:51.032005174Z\",\n      \"endTime\": \"2026-10-27T09:32:53.54866754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-nd4jp: Creating: virtualmachine creating or updating\\nMachine o3t1v0i5b5o7c5l-np-delete-main-bg4kp-q624c: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/083c3fe4-0c43-4b6a-9905-e8c1aa97e192\n            correlation ID: 01ff1c80-ae7b-4cd9-aeb7-68197786fff9\n            client request ID: unavailable","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3e7244a85da8bd67de02bd0b8b84b756179c64ce2823e309e63b548314c55c8e","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-4j5vtdb84h8v\" to finish creating: GET https://management"}},{"row_id":"6b72710c084e18db498a9dfb1f3fc208404e065371bfb1e2392d4d1776733d33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"5bf62fc718ddab8012057c407f628773cdec5fa1f4dce294092b41dd220fca46","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3baee6990108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2gdwf4pbz22s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"name\": \"0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:12:10.541346261Z\",\n      \"endTime\": \"2026-10-27T08:31:14.925265024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n            correlation ID: 6e36e413-6dec-4393-bcb3-4788f0d108b3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3baee6990108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2gdwf4pbz22s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"name\": \"0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:12:10.541346261Z\",\n      \"endTime\": \"2026-10-27T08:31:14.925265024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-ffmd5-h94sn: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-ffmd5-hvcp5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0e019fa7-e26d-44c3-b5c0-c1a3e7ecc4af\n            correlation ID: 6e36e413-6dec-4393-bcb3-4788f0d108b3\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"86a06a27f09769a51a2994045860d6d273e2c97dcb6ca43fae74008f3dc4c8e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"5d0bce7e827e2d05c2cc18c9afe55ae2811c60bd7230edf27e23c249ad468b73","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1401a4bfa220\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-76t2rcnvs776\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfb241d-404a-41b7-85ba-06ec88d78400\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfb241d-404a-41b7-85ba-06ec88d78400\",\n      \"name\": \"bcfb241d-404a-41b7-85ba-06ec88d78400\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:11:08.682187527Z\",\n      \"endTime\": \"2026-10-27T08:30:15.063678012Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r6n8p2x9f8z6q7e-np-1-x9ghg-6ql2f: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r6n8p2x9f8z6q7e-np-1-x9ghg-q2s94: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r6n8p2x9f8z6q7e-np-1-x9ghg-6ql2f: Creating: virtualmachine creating or updating\\nMachine r6n8p2x9f8z6q7e-np-1-x9ghg-q2s94: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfb241d-404a-41b7-85ba-06ec88d78400\n            correlation ID: d86dd47b-4014-46c8-b8ef-73967dae5ad5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1401a4bfa220\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-76t2rcnvs776\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfb241d-404a-41b7-85ba-06ec88d78400\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfb241d-404a-41b7-85ba-06ec88d78400\",\n      \"name\": \"bcfb241d-404a-41b7-85ba-06ec88d78400\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:11:08.682187527Z\",\n      \"endTime\": \"2026-10-27T08:30:15.063678012Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r6n8p2x9f8z6q7e-np-1-x9ghg-6ql2f: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r6n8p2x9f8z6q7e-np-1-x9ghg-q2s94: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r6n8p2x9f8z6q7e-np-1-x9ghg-6ql2f: Creating: virtualmachine creating or updating\\nMachine r6n8p2x9f8z6q7e-np-1-x9ghg-q2s94: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bcfb241d-404a-41b7-85ba-06ec88d78400\n            correlation ID: d86dd47b-4014-46c8-b8ef-73967dae5ad5\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"87b5adc7c377e1386a298a96203f79069411516584aeb8ed16d1f93968f56013","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"48f521861c358da162e56adb1d5a5560a6d110197a1e1ace87d26c0bd91ae752","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1520efb25aa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-g5mddw2znmwk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"name\": \"a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:30:17.391270547Z\",\n      \"endTime\": \"2026-10-27T08:49:18.18634688Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-gzw7k-gkpzc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v20261001-np-1-gzw7k-sxqb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-gzw7k-gkpzc: Creating: virtualmachine creating or updating\\nMachine v20261001-np-1-gzw7k-sxqb6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n            correlation ID: c7660621-545a-4ad8-89c0-9dd90d63841c\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:96]: failed to create node pool \"np-1\" for v20261001 cluster \"v20261001\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1520efb25aa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001-g5mddw2znmwk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"name\": \"a9c68b69-2f13-4771-88f2-779ee3c74191\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:30:17.391270547Z\",\n      \"endTime\": \"2026-10-27T08:49:18.18634688Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v20261001-np-1-gzw7k-gkpzc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v20261001-np-1-gzw7k-sxqb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v20261001-np-1-gzw7k-gkpzc: Creating: virtualmachine creating or updating\\nMachine v20261001-np-1-gzw7k-sxqb6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9c68b69-2f13-4771-88f2-779ee3c74191\n            correlation ID: c7660621-545a-4ad8-89c0-9dd90d63841c\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9c2487a391e154bc0cdb1e454eabd564d1391102d09f2bae0788296e90182f17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"7a2b3613d8cecf72125933f4f03d2d7d31c60d9319bc6e32bde74818fbf4a7c8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-fpbjtx\" for cluster \"cluster-candidate-4-21-fpbjtx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x324eadaf2a60\u003e: \n    failed to create NodePool np-fpbjtx: failed waiting for nodepool=\"np-fpbjtx\" for cluster \"cluster-candidate-4-21-fpbjtx\" in resourcegroup=\"rg-candidate-4-21-5f22mllfvqc4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a3967a25-cc4a-4063-9e34-8923a8b64863\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a3967a25-cc4a-4063-9e34-8923a8b64863\",\n      \"name\": \"a3967a25-cc4a-4063-9e34-8923a8b64863\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:33:18.660197992Z\",\n      \"endTime\": \"2026-10-27T08:52:18.18082794Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o2k6z2j0v1z5d4j-np-fpbjtx-lg9g5-dzsl5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o2k6z2j0v1z5d4j-np-fpbjtx-lg9g5-sv4wl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o2k6z2j0v1z5d4j-np-fpbjtx-lg9g5-dzsl5: Creating: virtualmachine creating or updating\\nMachine o2k6z2j0v1z5d4j-np-fpbjtx-lg9g5-sv4wl: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a3967a25-cc4a-4063-9e34-8923a8b64863\n            correlation ID: 9548c536-1f3f-4571-b9bb-a8ccda502489\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-fpbjtx\" for cluster \"cluster-candidate-4-21-fpbjtx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x324eadaf2a60\u003e: \n    failed to create NodePool np-fpbjtx: failed waiting for nodepool=\"np-fpbjtx\" for cluster \"cluster-candidate-4-21-fpbjtx\" in resourcegroup=\"rg-candidate-4-21-5f22mllfvqc4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a3967a25-cc4a-4063-9e34-8923a8b64863\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a3967a25-cc4a-4063-9e34-8923a8b64863\",\n      \"name\": \"a3967a25-cc4a-4063-9e34-8923a8b64863\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:33:18.660197992Z\",\n      \"endTime\": \"2026-10-27T08:52:18.18082794Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o2k6z2j0v1z5d4j-np-fpbjtx-lg9g5-dzsl5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o2k6z2j0v1z5d4j-np-fpbjtx-lg9g5-sv4wl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o2k6z2j0v1z5d4j-np-fpbjtx-lg9g5-dzsl5: Creating: virtualmachine creating or updating\\nMachine o2k6z2j0v1z5d4j-np-fpbjtx-lg9g5-sv4wl: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a3967a25-cc4a-4063-9e34-8923a8b64863\n            correlation ID: 9548c536-1f3f-4571-b9bb-a8ccda502489\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a8404f0821bff31c2157a15daa404a7d3eca57e2377c639eb0695b26950c6191","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"65786b9fd4ff52613cb0353320693acaf1aa58e9e4797f497abddc9ba6d5c20f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-2pmcg4\" for cluster \"cluster-candidate-4-22-2pmcg4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x144f200bc9a0\u003e: \n    failed to create NodePool np-2pmcg4: failed waiting for nodepool=\"np-2pmcg4\" for cluster \"cluster-candidate-4-22-2pmcg4\" in resourcegroup=\"rg-candidate-4-22-kwkd4shpnzkv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc208071-b38b-467a-86c5-b1a1983a3523\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc208071-b38b-467a-86c5-b1a1983a3523\",\n      \"name\": \"cc208071-b38b-467a-86c5-b1a1983a3523\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:34:38.028204768Z\",\n      \"endTime\": \"2026-10-27T08:53:38.318204864Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j6w2a7x0u1f2u0h-np-2pmcg4-vfsxq-2v2b5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j6w2a7x0u1f2u0h-np-2pmcg4-vfsxq-dmn7m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j6w2a7x0u1f2u0h-np-2pmcg4-vfsxq-2v2b5: Creating: virtualmachine creating or updating\\nMachine j6w2a7x0u1f2u0h-np-2pmcg4-vfsxq-dmn7m: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc208071-b38b-467a-86c5-b1a1983a3523\n            correlation ID: 27a80d85-0fd2-4584-9216-69874b20c19e\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-2pmcg4\" for cluster \"cluster-candidate-4-22-2pmcg4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x144f200bc9a0\u003e: \n    failed to create NodePool np-2pmcg4: failed waiting for nodepool=\"np-2pmcg4\" for cluster \"cluster-candidate-4-22-2pmcg4\" in resourcegroup=\"rg-candidate-4-22-kwkd4shpnzkv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc208071-b38b-467a-86c5-b1a1983a3523\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc208071-b38b-467a-86c5-b1a1983a3523\",\n      \"name\": \"cc208071-b38b-467a-86c5-b1a1983a3523\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:34:38.028204768Z\",\n      \"endTime\": \"2026-10-27T08:53:38.318204864Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j6w2a7x0u1f2u0h-np-2pmcg4-vfsxq-2v2b5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j6w2a7x0u1f2u0h-np-2pmcg4-vfsxq-dmn7m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j6w2a7x0u1f2u0h-np-2pmcg4-vfsxq-2v2b5: Creating: virtualmachine creating or updating\\nMachine j6w2a7x0u1f2u0h-np-2pmcg4-vfsxq-dmn7m: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc208071-b38b-467a-86c5-b1a1983a3523\n            correlation ID: 27a80d85-0fd2-4584-9216-69874b20c19e\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dfdfa53e2610bbf7d9ca663fea93f7ee40c18d419e2e0744d1dd8bf300e0c5e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"80f41f56d6bb15b31124a77cca8ab3bafdbb09260df5d6a0be927f35414248b0","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21bec3e60060\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-p48wwqxwbhds\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a694363-a54b-4c78-9450-e4038850db77\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a694363-a54b-4c78-9450-e4038850db77\",\n      \"name\": \"4a694363-a54b-4c78-9450-e4038850db77\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:09:27.098324224Z\",\n      \"endTime\": \"2026-10-27T08:28:35.105115624Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-n69xn-2rm9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-n69xn-2rm9g: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a694363-a54b-4c78-9450-e4038850db77\n            correlation ID: ae9de2e2-5408-48a4-abcb-54d1d6de37d5\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21bec3e60060\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-p48wwqxwbhds\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a694363-a54b-4c78-9450-e4038850db77\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a694363-a54b-4c78-9450-e4038850db77\",\n      \"name\": \"4a694363-a54b-4c78-9450-e4038850db77\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:09:27.098324224Z\",\n      \"endTime\": \"2026-10-27T08:28:35.105115624Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-n69xn-2rm9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-n69xn-2rm9g: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a694363-a54b-4c78-9450-e4038850db77\n            correlation ID: ae9de2e2-5408-48a4-abcb-54d1d6de37d5\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f03b6a72b00d53feeb2d88010ebcdbc9fa8467e617269c3729c9242c4e3c9a83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"13447a28327f6cac72eeb38ac4f64daeca2de13dce1a8779d5b023237c94cc0d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2089be556480\u003e: \n    failed to create NodePool np-jxwc9m: failed waiting for nodepool=\"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\" in resourcegroup=\"rg-candidate-5-1-t5bnkh9ptx8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"name\": \"aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:36:20.928028726Z\",\n      \"endTime\": \"2026-10-27T08:55:28.505111283Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: Creating: virtualmachine creating or updating\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n            correlation ID: 994d5b05-7775-48cd-93aa-f17d4c56d0ef\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:181]: failed to create node pool \"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2089be556480\u003e: \n    failed to create NodePool np-jxwc9m: failed waiting for nodepool=\"np-jxwc9m\" for cluster \"cluster-candidate-5-1-jxwc9m\" in resourcegroup=\"rg-candidate-5-1-t5bnkh9ptx8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"name\": \"aa13a852-544e-4ae1-9ee9-db477af0802a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:36:20.928028726Z\",\n      \"endTime\": \"2026-10-27T08:55:28.505111283Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-nmkn7: Creating: virtualmachine creating or updating\\nMachine f5e7a9i9o6b7x5n-np-jxwc9m-l25lf-rxvtq: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa13a852-544e-4ae1-9ee9-db477af0802a\n            correlation ID: 994d5b05-7775-48cd-93aa-f17d4c56d0ef\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fed0bdb4787e3dea3ff6c741ff8992fbdb130e6cad0751036b102a5e3b26bfa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114983782099456000","occurred_at":"2026-10-27T07:33:22Z","failed_at":"unknown","signature_id":"2a7dcb6ff7dcda026460bada2d10af749d2aaf152cdd767259012c980422e2c3","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel/all","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xa0fb7d04138\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-xf9jw4l5cdpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"name\": \"0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:14:54.971524126Z\",\n      \"endTime\": \"2026-10-27T08:33:58.162558969Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n            correlation ID: 7bfa7cc1-239c-4ae3-a9bc-578e94ecc5a3\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xa0fb7d04138\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-xf9jw4l5cdpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"name\": \"0006ad13-87f3-4fc4-8b43-20306abddf58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-27T08:14:54.971524126Z\",\n      \"endTime\": \"2026-10-27T08:33:58.162558969Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-7cl7p-8fh5m: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-7cl7p-fd22m: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0006ad13-87f3-4fc4-8b43-20306abddf58\n            correlation ID: 7bfa7cc1-239c-4ae3-a9bc-578e94ecc5a3\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ece5aa7e8b4a7df4eadead8c6c087ae24930024b2ab923cda14a7ad621f8a51","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["0ece5aa7e8b4a7df4eadead8c6c087ae24930024b2ab923cda14a7ad621f8a51","3e7244a85da8bd67de02bd0b8b84b756179c64ce2823e309e63b548314c55c8e","a62fe6e93c13b71d85f40e49c44ce5c550a3f71868c6f563af74c498e6155312"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all/2114918193188311040","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel-all","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-27T03:12:45Z","started_at":"2026-10-27T03:12:45Z","completed_at":"2026-10-27T06:06:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T06:12:45.67016066Z","timing_metadata_checked_at":"2026-10-27T06:12:45.67016066Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T06:12:45.67324795Z","region_metadata_checked_at":"2026-10-27T06:12:45.67324795Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-27T02:00:05Z","started_at":"2026-10-27T02:00:05Z","completed_at":"2026-10-27T03:54:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T04:14:52.560255575Z","timing_metadata_checked_at":"2026-10-27T04:14:52.560255575Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T04:14:52.564720461Z","region_metadata_checked_at":"2026-10-27T04:14:52.564720461Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"8d301747d1b4b72da017756e2967ffa5ed1cd9229fec37d57b58223bf6735155","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","failed_at":"e2e","signature_id":"cb8a33e2b2095072faf50f6e0575b49f9f24cba8d6e5696426aaf71bfa169c8e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-2nz4xc\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x203abb848000\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x203ab927a940\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:256]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-2nz4xc\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x203abb848000\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.001s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x203ab927a940\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ff366311fd43d72666297c0085142c8518628a4849f37fc8ec0089c5423ea5c2","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"9e0539c8c18f47b9b6ba104bab5a28922b76353417c81649f84b83043dd8b337","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","failed_at":"e2e","signature_id":"5f7305d0b3b91c61eb8e4f56e572507a452aa85cc00ec307edf3b8e9bfddebb6","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x66df03e8b20\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x66df03e8b20\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e7eacd17b93a9922f30449fc8ac0b2b6c2f322360a8ba998276c9e141732d710","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"be766c80fb4299fe62868a331c77d9670ed6ae0a85423f20286778c3dc598613","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","failed_at":"e2e","signature_id":"70755a49efebe5ef08cdb2d0691a83edd6fc97c800ce0e052ac99acb90b8d7f5","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2de6fa3020f0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:528]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2de6fa3020f0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e7eacd17b93a9922f30449fc8ac0b2b6c2f322360a8ba998276c9e141732d710","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"c05427536c33f23bd13fd8d0f7e9e559727e6d060c979264cf3a476288b27eb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2114899906945617920","occurred_at":"2026-10-27T02:00:05Z","failed_at":"e2e","signature_id":"e6435ae321fc270025d85586172e832697f963663729776cb49988ed3cb99186","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:260]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-23-wlxxtl\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x30c9514302d0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:260]: ACM must not create its ingress certificate secret for cluster \"cluster-candidate-4-23-wlxxtl\"\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x30c9514302d0\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e7eacd17b93a9922f30449fc8ac0b2b6c2f322360a8ba998276c9e141732d710","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["e7eacd17b93a9922f30449fc8ac0b2b6c2f322360a8ba998276c9e141732d710","ff366311fd43d72666297c0085142c8518628a4849f37fc8ec0089c5423ea5c2"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}