{"meta":{"date":"2026-10-02","timezone":"UTC","generated_at":"2026-10-29T21:25:04Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":6,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T23:00:44Z","started_at":"2026-10-02T23:00:44Z","completed_at":"2026-10-03T00:41:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T00:54:33.521888808Z","timing_metadata_checked_at":"2026-10-03T00:54:33.521888808Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T00:54:33.525661696Z","region_metadata_checked_at":"2026-10-03T00:54:33.525661696Z"},"failed_at":["e2e"],"failed_test_count":25,"occurrences":[{"row_id":"22a2fa267a120bfb0b960e647521d14d95bfa582d7976eecac20cef9ab1c69b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"a70f09283f83907ef445a20f2e96d80236d23d53ed875421b7b5e1da8a6a4b87","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-27sjww at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d81140\u003e: \n    failed to create HCP cluster np-skip-minor-27sjww: failed waiting for cluster=\"np-skip-minor-27sjww\" in resourcegroup=\"rg-np-skip-minor-27sjww-kgw6mbppvdmz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n      \"name\": \"74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:50:41.341932888Z\",\n      \"endTime\": \"2026-10-03T00:19:42.833586106Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-27sjww at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d81140\u003e: \n    failed to create HCP cluster np-skip-minor-27sjww: failed waiting for cluster=\"np-skip-minor-27sjww\" in resourcegroup=\"rg-np-skip-minor-27sjww-kgw6mbppvdmz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n      \"name\": \"74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:50:41.341932888Z\",\n      \"endTime\": \"2026-10-03T00:19:42.833586106Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"23fcefc5b11d65413ea804a8089bb9a1398ada3fb4ca364014a85387b12f549b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"f944baaa2c2ba95ce8132743ac5ce22cf1d0b2f92722f2569ba0b7a850637a81","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-8cm72gr5ql2s/cluster-nightly-4-22-65xxws should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00164c880\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-65xxws: failed waiting for cluster=\"cluster-nightly-4-22-65xxws\" in resourcegroup=\"rg-nightly-4-22-8cm72gr5ql2s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\",\n      \"name\": \"562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:28:43.258428047Z\",\n      \"endTime\": \"2026-10-02T23:57:43.738727793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-8cm72gr5ql2s/cluster-nightly-4-22-65xxws should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00164c880\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-65xxws: failed waiting for cluster=\"cluster-nightly-4-22-65xxws\" in resourcegroup=\"rg-nightly-4-22-8cm72gr5ql2s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\",\n      \"name\": \"562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:28:43.258428047Z\",\n      \"endTime\": \"2026-10-02T23:57:43.738727793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3877ae37640449584ed6f000dd6c4c3d6187efb87205f4490a43d2f9d61d1cf0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"8579cebca7275d6feeb1cb980486c433a32dc36619b5f2acd7985e3b37d192cf","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00106c720\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-88jnzt6h65n4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00106c720\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-88jnzt6h65n4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3e7429b3223a1d6d355e131e14bce8c79647455a8b75c865c3babe5f99cb7d3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"ec37fd56a0307f63cfac0b7076ce6db1a98af0f908355fdb720d73618014817c","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-766n78\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00149e390\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-qb9p7vsl4pjf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-766n78\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-766n78\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00149e390\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-qb9p7vsl4pjf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-766n78\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"41e0d49cf66aee5b2b9407196395cb8b5ed0b711efd641841a05a0ae84f7c1ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"3c45a70cfc1f78d78efba119b5ce6faa0b80d4c42bcb7dd0ee59de43bc516ac9","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00182a270\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7bxvbnqqpfz2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00182a270\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7bxvbnqqpfz2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4c1ab9d3c027b28656115e6dfb704765226717d072023348ee2328238134c91c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"1644bbcb2e6380ac66cea640064367161834cb558a92eb398848bad14721b31d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-p76rlg\" to 4.21.0-0.nightly-multi-2026-10-02-021649\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ab88a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-c4g7bpjwkcrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-p76rlg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-p76rlg\" to 4.21.0-0.nightly-multi-2026-10-02-021649\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ab88a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-c4g7bpjwkcrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-p76rlg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5d55c2aebec03b92ef9b0a8ea1bbdf42b86b5bc81071271009dd8de1b9f2424b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"cd2c3947df5446ae84ba6b5d7e021afccf38c12db58a8bee834290fd0af21ba2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tqrvbk\" for cluster \"cluster-nightly-5-0-tqrvbk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f91320\u003e: \n    failed to create NodePool np-tqrvbk: failed waiting for nodepool=\"np-tqrvbk\" for cluster \"cluster-nightly-5-0-tqrvbk\" in resourcegroup=\"rg-nightly-5-0-r6nmqb54mssw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/096e1e24-3a65-44de-9802-0a6339605f8f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/096e1e24-3a65-44de-9802-0a6339605f8f\",\n      \"name\": \"096e1e24-3a65-44de-9802-0a6339605f8f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:41:44.73235146Z\",\n      \"endTime\": \"2026-10-03T00:00:53.838500072Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-89fp6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-d94nh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-89fp6: Creating: virtualmachine creating or updating\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-d94nh: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tqrvbk\" for cluster \"cluster-nightly-5-0-tqrvbk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f91320\u003e: \n    failed to create NodePool np-tqrvbk: failed waiting for nodepool=\"np-tqrvbk\" for cluster \"cluster-nightly-5-0-tqrvbk\" in resourcegroup=\"rg-nightly-5-0-r6nmqb54mssw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/096e1e24-3a65-44de-9802-0a6339605f8f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/096e1e24-3a65-44de-9802-0a6339605f8f\",\n      \"name\": \"096e1e24-3a65-44de-9802-0a6339605f8f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:41:44.73235146Z\",\n      \"endTime\": \"2026-10-03T00:00:53.838500072Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-89fp6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-d94nh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-89fp6: Creating: virtualmachine creating or updating\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-d94nh: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"673cf86d98764226c7affd62e1847aa6911ede20e096bbfca7b0205b94acca77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"7eff813a5363b798d17a20887832fb9ba98d961ef78f14f728e98b256371020a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-lg5p6f\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a56dc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-lg5p6f: failed waiting for cluster=\"cp-ystream-upgrade-5-0-lg5p6f\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-b2ldx28lrgzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\",\n      \"name\": \"9c7f2efd-8245-4407-9270-f3937d54358b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:32:39.04806912Z\",\n      \"endTime\": \"2026-10-03T00:01:43.756113041Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-lg5p6f\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a56dc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-lg5p6f: failed waiting for cluster=\"cp-ystream-upgrade-5-0-lg5p6f\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-b2ldx28lrgzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\",\n      \"name\": \"9c7f2efd-8245-4407-9270-f3937d54358b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:32:39.04806912Z\",\n      \"endTime\": \"2026-10-03T00:01:43.756113041Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6a329dc7cda791050392c86be35e3a0986db401223546d6c837c6d0d9319b070","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"770ee469d83ba1d5d599e53e42dfd2f872e81b78bafe998012dfc850626289cc","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00141a900\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-kmfrpjr862hw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kmfrpjr862hw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00141a900\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-kmfrpjr862hw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kmfrpjr862hw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02985e80f9e6dd23af129635b4bd016a473a6a57b067e6f88bbc87e7f20a5595","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6afe1a99b1b8ac172632330a1e087e0d9d8a9abc157e6ffcdf220b2847251fe5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"860eb571af757ccc8cd08de2262a7ce14edf5abeb53618b97b6ce441d905813c","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010647a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-b4pbcj49h2gk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\",\n      \"name\": \"56464cc9-6bea-4036-8793-830b73f9c445\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:18:22.317106803Z\",\n      \"endTime\": \"2026-10-02T23:37:23.768654646Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: Creating: virtualmachine creating or updating\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010647a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-b4pbcj49h2gk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\",\n      \"name\": \"56464cc9-6bea-4036-8793-830b73f9c445\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:18:22.317106803Z\",\n      \"endTime\": \"2026-10-02T23:37:23.768654646Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: Creating: virtualmachine creating or updating\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6c02b4373162132fa81b41f63efab1c64e7c948ac700e39df02be7de2a79385a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"c7e8f4fd3f2bc41bffbc9f2d6790ff6e60abc16e8cbff179fedf0313fca2092e","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-85ljr8\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001476720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-85ljr8: failed waiting for cluster=\"cp-ystream-upgrade-4-23-85ljr8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xxm22554wqqw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4209a33e-d8fd-4e71-a9e2-6938fd555838\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4209a33e-d8fd-4e71-a9e2-6938fd555838\",\n      \"name\": \"4209a33e-d8fd-4e71-a9e2-6938fd555838\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:32:20.614033798Z\",\n      \"endTime\": \"2026-10-03T00:01:23.755103284Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-85ljr8\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001476720\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-85ljr8: failed waiting for cluster=\"cp-ystream-upgrade-4-23-85ljr8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xxm22554wqqw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4209a33e-d8fd-4e71-a9e2-6938fd555838\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4209a33e-d8fd-4e71-a9e2-6938fd555838\",\n      \"name\": \"4209a33e-d8fd-4e71-a9e2-6938fd555838\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:32:20.614033798Z\",\n      \"endTime\": \"2026-10-03T00:01:23.755103284Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"709d2057086c3f36e9914797fbf18ae41d1304e287bcc9470cc1a481ec31da7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"68ded3176e1d7fbcd1f8d33ace4e652b4594754aa12dcb2811b6d01aec17dc6b","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00100c018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-dbjg2z24dvcx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n      \"name\": \"2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:17:46.069336916Z\",\n      \"endTime\": \"2026-10-02T23:36:53.678900847Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-p829p-t26hm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-p829p-t26hm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00100c018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-dbjg2z24dvcx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n      \"name\": \"2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:17:46.069336916Z\",\n      \"endTime\": \"2026-10-02T23:36:53.678900847Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-p829p-t26hm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-p829p-t26hm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9580141697adb7a525d718295253254da6da1a71b022f3fab8ab2e393b4bec4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"8a7d51d9f85003ad4001a7f59ed5782eb2de4e62df81b0e16178abf5c822b1aa","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c80140\u003e: \n    failed to create NodePool np-fpzhbk: failed waiting for nodepool=\"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\" in resourcegroup=\"rg-nightly-4-21-lf2wqrcgx8mk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n      \"name\": \"f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:36:50.854597667Z\",\n      \"endTime\": \"2026-10-02T23:55:53.768834785Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: Creating: virtualmachine creating or updating\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c80140\u003e: \n    failed to create NodePool np-fpzhbk: failed waiting for nodepool=\"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\" in resourcegroup=\"rg-nightly-4-21-lf2wqrcgx8mk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n      \"name\": \"f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:36:50.854597667Z\",\n      \"endTime\": \"2026-10-02T23:55:53.768834785Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: Creating: virtualmachine creating or updating\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9d8ed537bdd276ef98cae12d894161ad6ebf274e3b8aec59fc813e10234a3bcb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"a22a4ddd3b8ff67faf5b163bf25b83cdba837b1c018fd882d7a9b40a6d220ea5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-gfwqg6\" to 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0018044b0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-p7jzbr5ffl22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-gfwqg6\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-gfwqg6\" to 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0018044b0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-p7jzbr5ffl22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-gfwqg6\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9e45ccb451c0609649fd3d064efdf69cadd76afe7e3ea1d4c3c423b9b3a4c195","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"088d80b26ce242a6a23ce42eb36d87da5a2ba3b0cb09615c4e35d5a5873f05c2","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001214a20\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-r44ftv\" in resourcegroup=\"rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-r44ftv/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001214a20\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-r44ftv\" in resourcegroup=\"rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-r44ftv/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a19bac4603b0b3d3aaa37dd3da9de7f22d558ee988b8b04f53e15bf86f5f4b26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"39ae3bb12ff8df998e369fcb228b237442fff1c7483c6d65c12668d7a03441bf","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0005e2c30\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9rcfwwnss4kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0005e2c30\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9rcfwwnss4kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a228a4819a47995e2dbb8d3d9ed55d86d6815032cdbce467c80f6d441a7efb1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"a6951ac14ce4de69f00809a996ba16b47c97b931a257020baf028934f05e2b52","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002910ff0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-rbgmrr9dbzzr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002910ff0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-rbgmrr9dbzzr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a9b92d48ea08eacd35c648a32cc39c2e9371ceb1e64080e239907db8bbf94b92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"5d19c65c5cc2f83977dfc0bb6da78485985e9a73a9516d73fcb59a9e71c1d6cf","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc002b293e0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rx57wpbdxhdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\",\n      \"name\": \"811670f5-5dbe-439c-9697-edae7243b031\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:47:39.993818098Z\",\n      \"endTime\": \"2026-10-03T00:06:42.823230365Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: Creating: virtualmachine creating or updating\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc002b293e0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rx57wpbdxhdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\",\n      \"name\": \"811670f5-5dbe-439c-9697-edae7243b031\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:47:39.993818098Z\",\n      \"endTime\": \"2026-10-03T00:06:42.823230365Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: Creating: virtualmachine creating or updating\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}},{"row_id":"bf7e43a787703e9d0e519cc491df6eff4feb9ca17052d660fe3071476d9a6512","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"428e779e55c222fac1f48387e160b1242c8a6e675443dd3caec7bdc747ed9c1c","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aae3e0\u003e: \n    failed to create HCP cluster np-dg-minor-t84576: failed waiting for cluster=\"np-dg-minor-t84576\" in resourcegroup=\"rg-np-dg-minor-t84576-xhl45nnpkcxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n      \"name\": \"86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:50:46.033712456Z\",\n      \"endTime\": \"2026-10-03T00:19:52.83457796Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aae3e0\u003e: \n    failed to create HCP cluster np-dg-minor-t84576: failed waiting for cluster=\"np-dg-minor-t84576\" in resourcegroup=\"rg-np-dg-minor-t84576-xhl45nnpkcxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n      \"name\": \"86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:50:46.033712456Z\",\n      \"endTime\": \"2026-10-03T00:19:52.83457796Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c1be8e717ca7e33e25aef11efb753f1e824cfcb0000c4b3412976f6a05202151","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"4c7eda445e675105debe286cdd5590eda5e7186b2f1edc9eaf4ca72af1eec7fa","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00089caa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-b7c2z62bpd6r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\",\n      \"name\": \"0c057ce9-73bf-4556-855a-010dcaa301e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:15:33.73277988Z\",\n      \"endTime\": \"2026-10-02T23:34:33.89328902Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-lcxp7-kcfwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-lcxp7-kcfwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00089caa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-b7c2z62bpd6r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\",\n      \"name\": \"0c057ce9-73bf-4556-855a-010dcaa301e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:15:33.73277988Z\",\n      \"endTime\": \"2026-10-02T23:34:33.89328902Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-lcxp7-kcfwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-lcxp7-kcfwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c9b61244728ec7adac538fde06974d1434af3a50004aa305ffe69f90c0545f0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"8eb7f7f722acb8f7d85fccf2574849a9b3348847ef0469070b23978413484f9f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vw9ss6\" for cluster \"cluster-nightly-4-23-vw9ss6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0f2c0\u003e: \n    failed to create NodePool np-vw9ss6: failed waiting for nodepool=\"np-vw9ss6\" for cluster \"cluster-nightly-4-23-vw9ss6\" in resourcegroup=\"rg-nightly-4-23-66zfk978pxch\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\",\n      \"name\": \"04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:42:21.716380971Z\",\n      \"endTime\": \"2026-10-03T00:01:23.832580289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-qhsr8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-wcqs9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-qhsr8: Creating: virtualmachine creating or updating\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-wcqs9: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vw9ss6\" for cluster \"cluster-nightly-4-23-vw9ss6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e0f2c0\u003e: \n    failed to create NodePool np-vw9ss6: failed waiting for nodepool=\"np-vw9ss6\" for cluster \"cluster-nightly-4-23-vw9ss6\" in resourcegroup=\"rg-nightly-4-23-66zfk978pxch\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\",\n      \"name\": \"04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:42:21.716380971Z\",\n      \"endTime\": \"2026-10-03T00:01:23.832580289Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-qhsr8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-wcqs9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-qhsr8: Creating: virtualmachine creating or updating\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-wcqs9: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c9bd3c052fc1970414f52ffb15a4cd3ebd9c34776d28d29908bf7e6dd35f2f8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"dfbe2f9b56ec8f589a40188c05855969f4988812afdbce974f8743866b87d6f5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rx699z\" for cluster \"cluster-nightly-5-1-rx699z\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eb8200\u003e: \n    failed to create NodePool np-rx699z: failed waiting for nodepool=\"np-rx699z\" for cluster \"cluster-nightly-5-1-rx699z\" in resourcegroup=\"rg-nightly-5-1-x6gpqlf2wmlk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ac35c1-1131-4bbe-9ed8-04f87748743e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ac35c1-1131-4bbe-9ed8-04f87748743e\",\n      \"name\": \"13ac35c1-1131-4bbe-9ed8-04f87748743e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:40:19.915190927Z\",\n      \"endTime\": \"2026-10-02T23:59:23.95618655Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-d28hs: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-kswmm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-d28hs: Creating: virtualmachine creating or updating\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-kswmm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rx699z\" for cluster \"cluster-nightly-5-1-rx699z\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eb8200\u003e: \n    failed to create NodePool np-rx699z: failed waiting for nodepool=\"np-rx699z\" for cluster \"cluster-nightly-5-1-rx699z\" in resourcegroup=\"rg-nightly-5-1-x6gpqlf2wmlk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ac35c1-1131-4bbe-9ed8-04f87748743e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ac35c1-1131-4bbe-9ed8-04f87748743e\",\n      \"name\": \"13ac35c1-1131-4bbe-9ed8-04f87748743e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:40:19.915190927Z\",\n      \"endTime\": \"2026-10-02T23:59:23.95618655Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-d28hs: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-kswmm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-d28hs: Creating: virtualmachine creating or updating\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-kswmm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cb3dfab3a1dc7e7676ad1d84e34d164aa7c286f8fc7cc2f6c813575fd3d25854","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"487336fbee3aa09886b1644e07daab686d89b729e7e6fdb636dc41d4196e0756","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001680330\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kcc2h87qnl8v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001680330\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kcc2h87qnl8v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d5776c6abb183a3f6781d5f0c5eaec1cfac5327b9912de6d9615e6bd221f1a20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"aae832d22fd2e270ae046b4cd1453e86b9c22535700ac5df9912205534e9a89c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be25e0\u003e: \n    failed to create NodePool np-f6fpxn: failed starting nodepool creation \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\" in resourcegroup=\"rg-nightly-4-20-9z2m2dsr8b8d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9z2m2dsr8b8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-f6fpxn/nodePools/np-f6fpxn\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be25e0\u003e: \n    failed to create NodePool np-f6fpxn: failed starting nodepool creation \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\" in resourcegroup=\"rg-nightly-4-20-9z2m2dsr8b8d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9z2m2dsr8b8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-f6fpxn/nodePools/np-f6fpxn\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"db203ed7bd9d663eede6d203c87152d9249c020886743e960789f814e255841a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"fc18079b3e233421ba88f574156106a7974d4521c46097383314a9ad391da2d6","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a36c0\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-tsckb2km58j7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-tsckb2km58j7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a36c0\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-tsckb2km58j7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-tsckb2km58j7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02985e80f9e6dd23af129635b4bd016a473a6a57b067e6f88bbc87e7f20a5595","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":25,"failure_pattern_ids":["02985e80f9e6dd23af129635b4bd016a473a6a57b067e6f88bbc87e7f20a5595","37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":25,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T20:18:59Z","started_at":"2026-10-02T20:18:59Z","completed_at":"2026-10-02T22:33:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T22:48:12.876088187Z","timing_metadata_checked_at":"2026-10-02T22:48:12.876088187Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T22:48:12.879316476Z","region_metadata_checked_at":"2026-10-02T22:48:12.879316476Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"096475a34230de023ffdf18480e79da893971abf21fa152226c5177638e4acfd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"4b3072ea271a5fa9392b9001a31d9bdf8953c887295f5949247942b3ff4ae13e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a95460\u003e: \n    failed to create NodePool np-k6xcnn: failed waiting for nodepool=\"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\" in resourcegroup=\"rg-candidate-5-1-xcbcjc2xvxnz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n      \"name\": \"f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:05:55.776603892Z\",\n      \"endTime\": \"2026-10-02T21:24:58.213196373Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a95460\u003e: \n    failed to create NodePool np-k6xcnn: failed waiting for nodepool=\"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\" in resourcegroup=\"rg-candidate-5-1-xcbcjc2xvxnz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n      \"name\": \"f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:05:55.776603892Z\",\n      \"endTime\": \"2026-10-02T21:24:58.213196373Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"40c7fe4c3256bcb09f65c2b610653c09d7451653e66201f04740388781b0c131","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"d53697eeb27bf8ab55313c9278e2e56de2e878d8f994cba4d1ca76149b70e248","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001880060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-flb2g5s6xxjr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c3afe0f8-5077-4a5f-b1fe-943343d11f7a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c3afe0f8-5077-4a5f-b1fe-943343d11f7a\",\n      \"name\": \"c3afe0f8-5077-4a5f-b1fe-943343d11f7a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:40:40.455432211Z\",\n      \"endTime\": \"2026-10-02T20:59:49.144387256Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-4kgx8-km75d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-4kgx8-wjql6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-4kgx8-km75d: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-4kgx8-wjql6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001880060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-flb2g5s6xxjr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c3afe0f8-5077-4a5f-b1fe-943343d11f7a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c3afe0f8-5077-4a5f-b1fe-943343d11f7a\",\n      \"name\": \"c3afe0f8-5077-4a5f-b1fe-943343d11f7a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:40:40.455432211Z\",\n      \"endTime\": \"2026-10-02T20:59:49.144387256Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-4kgx8-km75d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-4kgx8-wjql6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-4kgx8-km75d: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-4kgx8-wjql6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"49de21f41ff3ae1191602b48f25c5aa2e329b73b3beb517c5c54af7a669b45ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"250d803c2dde1e52d60a32aaa2d7a8f52b97c3de220de17e241c3f6720b1dab2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rfrkgm\" for cluster \"cluster-candidate-4-22-rfrkgm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0efa0\u003e: \n    failed to create NodePool np-rfrkgm: failed waiting for nodepool=\"np-rfrkgm\" for cluster \"cluster-candidate-4-22-rfrkgm\" in resourcegroup=\"rg-candidate-4-22-4dnctl8qsdmb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd66a171-5708-426c-a085-dc7fbb337a5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd66a171-5708-426c-a085-dc7fbb337a5f\",\n      \"name\": \"bd66a171-5708-426c-a085-dc7fbb337a5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:02:30.361632741Z\",\n      \"endTime\": \"2026-10-02T21:21:38.229702877Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-dgcdt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-z5pzp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-dgcdt: Creating: virtualmachine creating or updating\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-z5pzp: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rfrkgm\" for cluster \"cluster-candidate-4-22-rfrkgm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0efa0\u003e: \n    failed to create NodePool np-rfrkgm: failed waiting for nodepool=\"np-rfrkgm\" for cluster \"cluster-candidate-4-22-rfrkgm\" in resourcegroup=\"rg-candidate-4-22-4dnctl8qsdmb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd66a171-5708-426c-a085-dc7fbb337a5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd66a171-5708-426c-a085-dc7fbb337a5f\",\n      \"name\": \"bd66a171-5708-426c-a085-dc7fbb337a5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:02:30.361632741Z\",\n      \"endTime\": \"2026-10-02T21:21:38.229702877Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-dgcdt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-z5pzp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-dgcdt: Creating: virtualmachine creating or updating\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-z5pzp: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6bff5f7c3c8571a0d959d33c1b3491dc6e9e47b84700f612570d06237c20a164","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"2b86c91a6a4b689be874849e9832db0f4b59836029e6d1ac55b1fd9ebea622cc","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbb780\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-9v5cc2pldl4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\",\n      \"name\": \"64c53475-70e9-4ead-ac9d-b255c14e111e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:37:10.986126021Z\",\n      \"endTime\": \"2026-10-02T20:56:19.371811745Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbb780\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-9v5cc2pldl4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\",\n      \"name\": \"64c53475-70e9-4ead-ac9d-b255c14e111e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:37:10.986126021Z\",\n      \"endTime\": \"2026-10-02T20:56:19.371811745Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7094f0d1bcc262750230fc7e73625955c623a7bfa01e60ad26f60f32c7406359","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"04ae0703900792edf19aad7f285f7ea250c1b64e19621d532ae6fed3c83dba96","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c2b720\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fgzfw6brhrkb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n      \"name\": \"6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:34:15.426377086Z\",\n      \"endTime\": \"2026-10-02T21:53:18.348312497Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: Creating: virtualmachine creating or updating\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c2b720\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fgzfw6brhrkb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n      \"name\": \"6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:34:15.426377086Z\",\n      \"endTime\": \"2026-10-02T21:53:18.348312497Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: Creating: virtualmachine creating or updating\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}},{"row_id":"7fc7074a2cc6bc51764b983c068b6184b7facdf76df5394955c08f82f22416f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"f1c205eb5adab7b42be8bb2e5810e6b3d460997323e06bef104b859fa7c1c1fc","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f04d60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-j6tfrrstctwj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2237f1ea-36ac-4925-ab4c-3bf34b150641\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2237f1ea-36ac-4925-ab4c-3bf34b150641\",\n      \"name\": \"2237f1ea-36ac-4925-ab4c-3bf34b150641\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:40:29.897644584Z\",\n      \"endTime\": \"2026-10-02T20:59:29.402047255Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-krs6w-gdbz8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-krs6w-gdbz8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f04d60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-j6tfrrstctwj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2237f1ea-36ac-4925-ab4c-3bf34b150641\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2237f1ea-36ac-4925-ab4c-3bf34b150641\",\n      \"name\": \"2237f1ea-36ac-4925-ab4c-3bf34b150641\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:40:29.897644584Z\",\n      \"endTime\": \"2026-10-02T20:59:29.402047255Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-krs6w-gdbz8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-krs6w-gdbz8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"809b3ca47a0465dfb83498ae46c9bace135a5d4f0c91d5c0ef4aa82ba3138f06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"557730ef8a8a538ac02f9730574bc105fe419054c1342595ab60e5a9b836a55b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qpmglj\" for cluster \"cluster-candidate-4-20-qpmglj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00128af20\u003e: \n    failed to create NodePool np-qpmglj: failed waiting for nodepool=\"np-qpmglj\" for cluster \"cluster-candidate-4-20-qpmglj\" in resourcegroup=\"rg-candidate-4-20-lrjrfzqq46lq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f59c352-b15a-44fe-8eb1-052155b6abc3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f59c352-b15a-44fe-8eb1-052155b6abc3\",\n      \"name\": \"8f59c352-b15a-44fe-8eb1-052155b6abc3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:01:08.060624103Z\",\n      \"endTime\": \"2026-10-02T21:20:08.251497411Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-pngl6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-rh562: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-pngl6: Creating: virtualmachine creating or updating\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-rh562: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qpmglj\" for cluster \"cluster-candidate-4-20-qpmglj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00128af20\u003e: \n    failed to create NodePool np-qpmglj: failed waiting for nodepool=\"np-qpmglj\" for cluster \"cluster-candidate-4-20-qpmglj\" in resourcegroup=\"rg-candidate-4-20-lrjrfzqq46lq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f59c352-b15a-44fe-8eb1-052155b6abc3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f59c352-b15a-44fe-8eb1-052155b6abc3\",\n      \"name\": \"8f59c352-b15a-44fe-8eb1-052155b6abc3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:01:08.060624103Z\",\n      \"endTime\": \"2026-10-02T21:20:08.251497411Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-pngl6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-rh562: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-pngl6: Creating: virtualmachine creating or updating\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-rh562: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"997ea6840b66749ea36a8cd61c6485f99287d70f683f1fb6299a3abff3b69eba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"3ccc455f5000d71ad9adb6c80c8655900d46f92c09932844cfe825c30dea8f29","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001172048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-5sg2h42pm8fv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n      \"name\": \"4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:43:49.738242165Z\",\n      \"endTime\": \"2026-10-02T21:02:49.398233298Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001172048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-5sg2h42pm8fv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n      \"name\": \"4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:43:49.738242165Z\",\n      \"endTime\": \"2026-10-02T21:02:49.398233298Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"a4db154fd9c0a797d2debd5028cb127452841c9df00912c6d836549ef1804ad0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"abdd7c859f25fcc9e0e3eca9fc927795be2131c3357811538ef9ccf29668a7b0","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6f0a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-cccnsncfs2r4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c87a1bea-d45c-40f0-b36e-76c1418b3181\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c87a1bea-d45c-40f0-b36e-76c1418b3181\",\n      \"name\": \"c87a1bea-d45c-40f0-b36e-76c1418b3181\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:39:19.772177296Z\",\n      \"endTime\": \"2026-10-02T20:58:19.197731682Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-mcr5z: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-pn7ft: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-mcr5z: Creating: virtualmachine creating or updating\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-pn7ft: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6f0a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-cccnsncfs2r4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c87a1bea-d45c-40f0-b36e-76c1418b3181\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c87a1bea-d45c-40f0-b36e-76c1418b3181\",\n      \"name\": \"c87a1bea-d45c-40f0-b36e-76c1418b3181\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:39:19.772177296Z\",\n      \"endTime\": \"2026-10-02T20:58:19.197731682Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-mcr5z: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-pn7ft: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-mcr5z: Creating: virtualmachine creating or updating\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-pn7ft: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d5504a933d38e17db69389c3b78d3c03fac9b8947587ff7c8791691a178db9ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"df71d9af24f4d024cb085b025d0e7702e69475d8342378c7ed71b98fdfda61a5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4rsvdk\" for cluster \"cluster-candidate-5-0-4rsvdk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ced660\u003e: \n    failed to create NodePool np-4rsvdk: failed waiting for nodepool=\"np-4rsvdk\" for cluster \"cluster-candidate-5-0-4rsvdk\" in resourcegroup=\"rg-candidate-5-0-mr9dnz4m68mv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe99a24d-45b0-4c02-b399-8a58c33f760f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe99a24d-45b0-4c02-b399-8a58c33f760f\",\n      \"name\": \"fe99a24d-45b0-4c02-b399-8a58c33f760f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:03:59.509943483Z\",\n      \"endTime\": \"2026-10-02T21:23:08.337275316Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gjgrx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gkprg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gjgrx: Creating: virtualmachine creating or updating\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gkprg: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4rsvdk\" for cluster \"cluster-candidate-5-0-4rsvdk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ced660\u003e: \n    failed to create NodePool np-4rsvdk: failed waiting for nodepool=\"np-4rsvdk\" for cluster \"cluster-candidate-5-0-4rsvdk\" in resourcegroup=\"rg-candidate-5-0-mr9dnz4m68mv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe99a24d-45b0-4c02-b399-8a58c33f760f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe99a24d-45b0-4c02-b399-8a58c33f760f\",\n      \"name\": \"fe99a24d-45b0-4c02-b399-8a58c33f760f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:03:59.509943483Z\",\n      \"endTime\": \"2026-10-02T21:23:08.337275316Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gjgrx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gkprg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gjgrx: Creating: virtualmachine creating or updating\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gkprg: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d7ec549d4a5026535a9f1f3502a62dff432dfd2420632ef356b55fa7d09174b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"7ed334e95fee4ff7d071b052175951829e19af47fe9ec8e38db681d9e969f735","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d194a0\u003e: \n    failed to create NodePool np-7lqqwg: failed waiting for nodepool=\"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\" in resourcegroup=\"rg-candidate-4-21-fsj5jdg9tljp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\",\n      \"name\": \"d395f2bf-88ca-4f84-8998-c863978caf3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:03:18.868079791Z\",\n      \"endTime\": \"2026-10-02T21:22:18.599909683Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d194a0\u003e: \n    failed to create NodePool np-7lqqwg: failed waiting for nodepool=\"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\" in resourcegroup=\"rg-candidate-4-21-fsj5jdg9tljp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\",\n      \"name\": \"d395f2bf-88ca-4f84-8998-c863978caf3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:03:18.868079791Z\",\n      \"endTime\": \"2026-10-02T21:22:18.599909683Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ece2ff7cc8c9394af8ab3d38412290220d0120ec136fc1d6dafe1061e92efc4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"6a6e10696b7a19366e889f690b2ab374029b36eaa72533a7417b96b82b33497e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-878fr2\" for cluster \"cluster-candidate-4-23-878fr2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bb09a0\u003e: \n    failed to create NodePool np-878fr2: failed waiting for nodepool=\"np-878fr2\" for cluster \"cluster-candidate-4-23-878fr2\" in resourcegroup=\"rg-candidate-4-23-tq4tlj8sn8rt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d39da2e3-e8ed-4ecb-a802-520caf1d680a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d39da2e3-e8ed-4ecb-a802-520caf1d680a\",\n      \"name\": \"d39da2e3-e8ed-4ecb-a802-520caf1d680a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:05:40.949801842Z\",\n      \"endTime\": \"2026-10-02T21:24:48.213673897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-7czvr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-wcjsm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-7czvr: Creating: virtualmachine creating or updating\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-wcjsm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-878fr2\" for cluster \"cluster-candidate-4-23-878fr2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bb09a0\u003e: \n    failed to create NodePool np-878fr2: failed waiting for nodepool=\"np-878fr2\" for cluster \"cluster-candidate-4-23-878fr2\" in resourcegroup=\"rg-candidate-4-23-tq4tlj8sn8rt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d39da2e3-e8ed-4ecb-a802-520caf1d680a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d39da2e3-e8ed-4ecb-a802-520caf1d680a\",\n      \"name\": \"d39da2e3-e8ed-4ecb-a802-520caf1d680a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:05:40.949801842Z\",\n      \"endTime\": \"2026-10-02T21:24:48.213673897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-7czvr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-wcjsm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-7czvr: Creating: virtualmachine creating or updating\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-wcjsm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T17:45:08Z","started_at":"2026-10-02T17:45:08Z","completed_at":"2026-10-02T20:14:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T20:31:44.030568391Z","timing_metadata_checked_at":"2026-10-02T20:31:44.030568391Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T20:31:44.033696681Z","region_metadata_checked_at":"2026-10-02T20:31:44.033696681Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"06b39523714f8e8b897e0de9dba735e4495bd8afb0f523c0098db71c4a357721","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"f3c82f6a3a610eb2845b45bce009911813c9de13006d2bd6433c177db3846841","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee2740\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-ht5gkd\" in resourcegroup=\"rg-np-noedge-ht5gkd-s9tjwqwzr9v2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2685dbe2-74b7-44fa-8866-0542656a2cbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2685dbe2-74b7-44fa-8866-0542656a2cbe\",\n      \"name\": \"2685dbe2-74b7-44fa-8866-0542656a2cbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T19:26:52.969260057Z\",\n      \"endTime\": \"2026-10-02T19:45:54.294162813Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-ht5gkd-s9tjwqwzr9v2-np-ne-ht5gkd--managed/providers/Microsoft.Compute/virtualMachines/s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-ht5gkd-s9tjwqwzr9v2-np-ne-ht5gkd--managed/providers/Microsoft.Compute/virtualMachines/s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ee2740\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-ht5gkd\" in resourcegroup=\"rg-np-noedge-ht5gkd-s9tjwqwzr9v2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2685dbe2-74b7-44fa-8866-0542656a2cbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2685dbe2-74b7-44fa-8866-0542656a2cbe\",\n      \"name\": \"2685dbe2-74b7-44fa-8866-0542656a2cbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T19:26:52.969260057Z\",\n      \"endTime\": \"2026-10-02T19:45:54.294162813Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-ht5gkd-s9tjwqwzr9v2-np-ne-ht5gkd--managed/providers/Microsoft.Compute/virtualMachines/s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-ht5gkd-s9tjwqwzr9v2-np-ne-ht5gkd--managed/providers/Microsoft.Compute/virtualMachines/s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"305355cd1672855345c55c9cb833c281f5b35129e59e38a0342d0122286831a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"0f05a349b9fb3a19a653fac755019ca3729cea73f673115f377ea1ff53ca460a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-97jnb4\" for cluster \"cluster-candidate-4-22-97jnb4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a61960\u003e: \n    failed to create NodePool np-97jnb4: failed waiting for nodepool=\"np-97jnb4\" for cluster \"cluster-candidate-4-22-97jnb4\" in resourcegroup=\"rg-candidate-4-22-gqvl8n7h4bmq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d14fd998-aa3e-49c8-a405-91cc8c02a377\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d14fd998-aa3e-49c8-a405-91cc8c02a377\",\n      \"name\": \"d14fd998-aa3e-49c8-a405-91cc8c02a377\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:49:32.578309403Z\",\n      \"endTime\": \"2026-10-02T19:08:33.241090855Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-dkh4p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-hz2bh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-dkh4p: Creating: virtualmachine creating or updating\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-hz2bh: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-97jnb4\" for cluster \"cluster-candidate-4-22-97jnb4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a61960\u003e: \n    failed to create NodePool np-97jnb4: failed waiting for nodepool=\"np-97jnb4\" for cluster \"cluster-candidate-4-22-97jnb4\" in resourcegroup=\"rg-candidate-4-22-gqvl8n7h4bmq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d14fd998-aa3e-49c8-a405-91cc8c02a377\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d14fd998-aa3e-49c8-a405-91cc8c02a377\",\n      \"name\": \"d14fd998-aa3e-49c8-a405-91cc8c02a377\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:49:32.578309403Z\",\n      \"endTime\": \"2026-10-02T19:08:33.241090855Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-dkh4p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-hz2bh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-dkh4p: Creating: virtualmachine creating or updating\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-hz2bh: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"57b017b159e60c7c24363451d6c46408f0d20e1ce34b2b43d09853443dad8f36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"23c88eb451a5cdd79574abe8ef9812935c0e0d4c234696f73be4ca6bce4d6bdc","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001522060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nl8vdbhxwtvx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n      \"name\": \"31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:27:19.35370587Z\",\n      \"endTime\": \"2026-10-02T18:46:22.521838138Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001522060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nl8vdbhxwtvx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n      \"name\": \"31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:27:19.35370587Z\",\n      \"endTime\": \"2026-10-02T18:46:22.521838138Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"6a4feb2d0892c04953ef2c114e0b49c03fb8abe3c287f3e70a7cdd4dd7858f11","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"63e9790a996ec967d16e002977f953d1b88bb5b5420740a8c85e4a4ad4676d27","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kfvn25\" for cluster \"cluster-candidate-4-21-kfvn25\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b3b220\u003e: \n    failed to create NodePool np-kfvn25: failed waiting for nodepool=\"np-kfvn25\" for cluster \"cluster-candidate-4-21-kfvn25\" in resourcegroup=\"rg-candidate-4-21-85wnrgqqbd2h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b12e98e2-7f93-4004-87fa-b863b0f0044f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b12e98e2-7f93-4004-87fa-b863b0f0044f\",\n      \"name\": \"b12e98e2-7f93-4004-87fa-b863b0f0044f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:49:07.875106668Z\",\n      \"endTime\": \"2026-10-02T19:08:13.588598197Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-f65j4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-g5bl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-f65j4: Creating: virtualmachine creating or updating\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-g5bl8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kfvn25\" for cluster \"cluster-candidate-4-21-kfvn25\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b3b220\u003e: \n    failed to create NodePool np-kfvn25: failed waiting for nodepool=\"np-kfvn25\" for cluster \"cluster-candidate-4-21-kfvn25\" in resourcegroup=\"rg-candidate-4-21-85wnrgqqbd2h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b12e98e2-7f93-4004-87fa-b863b0f0044f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b12e98e2-7f93-4004-87fa-b863b0f0044f\",\n      \"name\": \"b12e98e2-7f93-4004-87fa-b863b0f0044f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:49:07.875106668Z\",\n      \"endTime\": \"2026-10-02T19:08:13.588598197Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-f65j4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-g5bl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-f65j4: Creating: virtualmachine creating or updating\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-g5bl8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7a70a53372f675de9d9be7c3a4f423e6c619c728d3dcba789f1f378cf1efba0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"f9e948ed90e8383bba7bd2c174ac649975b4723ac3ec4ecc8f0a4e9d83325283","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vvnbz4\" for cluster \"cluster-candidate-4-20-vvnbz4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c04c80\u003e: \n    failed to create NodePool np-vvnbz4: failed waiting for nodepool=\"np-vvnbz4\" for cluster \"cluster-candidate-4-20-vvnbz4\" in resourcegroup=\"rg-candidate-4-20-6rrzp6smfk8r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a84f10bd-1000-4ff5-b696-ec0471f172d5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a84f10bd-1000-4ff5-b696-ec0471f172d5\",\n      \"name\": \"a84f10bd-1000-4ff5-b696-ec0471f172d5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:45:19.007012696Z\",\n      \"endTime\": \"2026-10-02T19:04:23.174354297Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-jqrxn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-rd5xk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-jqrxn: Creating: virtualmachine creating or updating\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-rd5xk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vvnbz4\" for cluster \"cluster-candidate-4-20-vvnbz4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c04c80\u003e: \n    failed to create NodePool np-vvnbz4: failed waiting for nodepool=\"np-vvnbz4\" for cluster \"cluster-candidate-4-20-vvnbz4\" in resourcegroup=\"rg-candidate-4-20-6rrzp6smfk8r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a84f10bd-1000-4ff5-b696-ec0471f172d5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a84f10bd-1000-4ff5-b696-ec0471f172d5\",\n      \"name\": \"a84f10bd-1000-4ff5-b696-ec0471f172d5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:45:19.007012696Z\",\n      \"endTime\": \"2026-10-02T19:04:23.174354297Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-jqrxn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-rd5xk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-jqrxn: Creating: virtualmachine creating or updating\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-rd5xk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7e8a6452e1d5372e6a4a55264eda0a7050b31b32932b66ce044aa5cceef70926","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"94dbc4ea2092ec4f2334ecba4369c328eb3548419ec0eafdb59df728c566ad14","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qv99kb\" for cluster \"cluster-candidate-4-23-qv99kb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c78ba0\u003e: \n    failed to create NodePool np-qv99kb: failed waiting for nodepool=\"np-qv99kb\" for cluster \"cluster-candidate-4-23-qv99kb\" in resourcegroup=\"rg-candidate-4-23-xhtfbmhw8nkt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93ec2ac0-8140-4df7-861f-8f5e69bd2825\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93ec2ac0-8140-4df7-861f-8f5e69bd2825\",\n      \"name\": \"93ec2ac0-8140-4df7-861f-8f5e69bd2825\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:47:56.273213854Z\",\n      \"endTime\": \"2026-10-02T19:07:03.234946252Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-nhvhz: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-xq4s5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-nhvhz: Creating: virtualmachine creating or updating\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-xq4s5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qv99kb\" for cluster \"cluster-candidate-4-23-qv99kb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c78ba0\u003e: \n    failed to create NodePool np-qv99kb: failed waiting for nodepool=\"np-qv99kb\" for cluster \"cluster-candidate-4-23-qv99kb\" in resourcegroup=\"rg-candidate-4-23-xhtfbmhw8nkt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93ec2ac0-8140-4df7-861f-8f5e69bd2825\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93ec2ac0-8140-4df7-861f-8f5e69bd2825\",\n      \"name\": \"93ec2ac0-8140-4df7-861f-8f5e69bd2825\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:47:56.273213854Z\",\n      \"endTime\": \"2026-10-02T19:07:03.234946252Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-nhvhz: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-xq4s5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-nhvhz: Creating: virtualmachine creating or updating\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-xq4s5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"962349f04f77c21cfacbfb650160ee9b7d7b5c8a1ef11f41409afabe6c7bf8bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"ff0f4f85405eb058ab255730c9bfcc4dfebb9a6c55cbfe3ed6cc4e8bfb37dbfd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xf2nmx\" for cluster \"cluster-candidate-5-0-xf2nmx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000995320\u003e: \n    failed to create NodePool np-xf2nmx: failed waiting for nodepool=\"np-xf2nmx\" for cluster \"cluster-candidate-5-0-xf2nmx\" in resourcegroup=\"rg-candidate-5-0-7jr6wdpsfvz4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\",\n      \"name\": \"992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:49:51.032351879Z\",\n      \"endTime\": \"2026-10-02T19:08:53.234722812Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-kqt4r: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-nwmrg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-kqt4r: Creating: virtualmachine creating or updating\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-nwmrg: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xf2nmx\" for cluster \"cluster-candidate-5-0-xf2nmx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000995320\u003e: \n    failed to create NodePool np-xf2nmx: failed waiting for nodepool=\"np-xf2nmx\" for cluster \"cluster-candidate-5-0-xf2nmx\" in resourcegroup=\"rg-candidate-5-0-7jr6wdpsfvz4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\",\n      \"name\": \"992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:49:51.032351879Z\",\n      \"endTime\": \"2026-10-02T19:08:53.234722812Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-kqt4r: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-nwmrg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-kqt4r: Creating: virtualmachine creating or updating\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-nwmrg: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"96aedd7bfda0362383c718c4ea76002c1bb06fb6573d1a07bf460124da373926","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"1991e4120c3a9404f4a08486e54a3b094270ef79c075deb389ccb4f87dd19186","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fcce00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-kndcxqxwg5m4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9940d584-67e5-4b2a-939b-305a5a782054\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9940d584-67e5-4b2a-939b-305a5a782054\",\n      \"name\": \"9940d584-67e5-4b2a-939b-305a5a782054\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:26:54.435762769Z\",\n      \"endTime\": \"2026-10-02T18:46:02.862827665Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-6stxx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-fvklg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-6stxx: Creating: virtualmachine creating or updating\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-fvklg: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fcce00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-kndcxqxwg5m4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9940d584-67e5-4b2a-939b-305a5a782054\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9940d584-67e5-4b2a-939b-305a5a782054\",\n      \"name\": \"9940d584-67e5-4b2a-939b-305a5a782054\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:26:54.435762769Z\",\n      \"endTime\": \"2026-10-02T18:46:02.862827665Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-6stxx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-fvklg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-6stxx: Creating: virtualmachine creating or updating\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-fvklg: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"afb7c28584b2497ec163cbad4f11b2f5049c464b9c15a9992bc4100ad2f35ea8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"c795eac422f14a1d4cf2c9186a4a74089ac3865e0cf6aad349163a72b03db067","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lvw97g\" for cluster \"cluster-candidate-5-1-lvw97g\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e66f00\u003e: \n    failed to create NodePool np-lvw97g: failed waiting for nodepool=\"np-lvw97g\" for cluster \"cluster-candidate-5-1-lvw97g\" in resourcegroup=\"rg-candidate-5-1-qrgrjlf6rkm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4a97c7bb-57af-45b7-8fb7-5b981cf7570f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4a97c7bb-57af-45b7-8fb7-5b981cf7570f\",\n      \"name\": \"4a97c7bb-57af-45b7-8fb7-5b981cf7570f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:50:51.431353811Z\",\n      \"endTime\": \"2026-10-02T19:09:53.217243745Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-95ljv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-cfkc9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-95ljv: Creating: virtualmachine creating or updating\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-cfkc9: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lvw97g\" for cluster \"cluster-candidate-5-1-lvw97g\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e66f00\u003e: \n    failed to create NodePool np-lvw97g: failed waiting for nodepool=\"np-lvw97g\" for cluster \"cluster-candidate-5-1-lvw97g\" in resourcegroup=\"rg-candidate-5-1-qrgrjlf6rkm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4a97c7bb-57af-45b7-8fb7-5b981cf7570f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4a97c7bb-57af-45b7-8fb7-5b981cf7570f\",\n      \"name\": \"4a97c7bb-57af-45b7-8fb7-5b981cf7570f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:50:51.431353811Z\",\n      \"endTime\": \"2026-10-02T19:09:53.217243745Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-95ljv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-cfkc9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-95ljv: Creating: virtualmachine creating or updating\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-cfkc9: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b0e4cbb8c8cd579f631dab696b910ba233dafd020215195a0d44279abd496b37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"d96bba282a27aaa6fbc92c0663dfd8f01103b1e67a58a200c8d1a2d1b8e4e832","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c9c280\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-gg8s9j\" in resourcegroup=\"rg-np-downgrade-gg8s9j-fnkpnt9xd47t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\",\n      \"name\": \"a4186f06-3e4f-4125-8780-405aa920bd3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T19:29:21.050247186Z\",\n      \"endTime\": \"2026-10-02T19:48:24.533281016Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c9c280\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-gg8s9j\" in resourcegroup=\"rg-np-downgrade-gg8s9j-fnkpnt9xd47t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\",\n      \"name\": \"a4186f06-3e4f-4125-8780-405aa920bd3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T19:29:21.050247186Z\",\n      \"endTime\": \"2026-10-02T19:48:24.533281016Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ccf7cf35d852b5a080806ae3c01fd13af18181e686ea23cca7840c454a70b752","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"7f3920bc72425d6da0c2994e526b54c2ba9fd9d44a6c8cb1fdea88ed4dae4ca0","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001085220\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-dk96qxfcglkz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a2c0d616-d3e1-4467-a097-edfb164f91f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a2c0d616-d3e1-4467-a097-edfb164f91f8\",\n      \"name\": \"a2c0d616-d3e1-4467-a097-edfb164f91f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:26:30.147930745Z\",\n      \"endTime\": \"2026-10-02T18:45:32.745630715Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-jbbww-kglw7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-jbbww-kglw7: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001085220\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-dk96qxfcglkz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a2c0d616-d3e1-4467-a097-edfb164f91f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a2c0d616-d3e1-4467-a097-edfb164f91f8\",\n      \"name\": \"a2c0d616-d3e1-4467-a097-edfb164f91f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:26:30.147930745Z\",\n      \"endTime\": \"2026-10-02T18:45:32.745630715Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-jbbww-kglw7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-jbbww-kglw7: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"df313438a992a9865e2096234146bb071b0155b14d8f0c1c25d1b9026c360484","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"1e6f9fa59cd98bfc284387b5af1ed56122fcb8e577d42d119bd60d5ef8e20605","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00341d8a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8s9pgvgwlgbj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n      \"name\": \"e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T19:20:19.695876004Z\",\n      \"endTime\": \"2026-10-02T19:39:24.271020967Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: Creating: virtualmachine creating or updating\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00341d8a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8s9pgvgwlgbj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n      \"name\": \"e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T19:20:19.695876004Z\",\n      \"endTime\": \"2026-10-02T19:39:24.271020967Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: Creating: virtualmachine creating or updating\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}},{"row_id":"e0481009c3ee19e3b6efac1ad603e143030ff4cb3f5b1ca21eb16f4696b9d31f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"3beeb059ed2eee1c225e5fa6e52e389772213e9dbefa4285fead2b9c61d38ee4","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001f12060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5x984bppgxmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f889fa2b-1853-4b53-b0a5-8af8bb1664c4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f889fa2b-1853-4b53-b0a5-8af8bb1664c4\",\n      \"name\": \"f889fa2b-1853-4b53-b0a5-8af8bb1664c4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:28:21.353402223Z\",\n      \"endTime\": \"2026-10-02T18:47:22.523072731Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-rvbvq-47m2n: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-rvbvq-dncqc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-rvbvq-47m2n: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-rvbvq-dncqc: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001f12060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5x984bppgxmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f889fa2b-1853-4b53-b0a5-8af8bb1664c4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f889fa2b-1853-4b53-b0a5-8af8bb1664c4\",\n      \"name\": \"f889fa2b-1853-4b53-b0a5-8af8bb1664c4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:28:21.353402223Z\",\n      \"endTime\": \"2026-10-02T18:47:22.523072731Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-rvbvq-47m2n: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-rvbvq-dncqc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-rvbvq-47m2n: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-rvbvq-dncqc: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T07:30:54Z","started_at":"2026-10-02T07:30:54Z","completed_at":"2026-10-02T09:51:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T10:00:10.31942121Z","timing_metadata_checked_at":"2026-10-02T10:00:10.31942121Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T10:00:10.322654298Z","region_metadata_checked_at":"2026-10-02T10:00:10.322654298Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"1491c913fd3907211e6e3fadfd6ec8cf83a8d0a9bbdcd73c6b7f5ebd107a8b8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"417e38533ea85fbd583e5552fc77a77554d65e5a8dd9d15343bf93a32380c555","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9d580\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-vrplznt5v54r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85d24adc-6f21-47c1-8812-f58f8001bb49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85d24adc-6f21-47c1-8812-f58f8001bb49\",\n      \"name\": \"85d24adc-6f21-47c1-8812-f58f8001bb49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:56:13.127412921Z\",\n      \"endTime\": \"2026-10-02T08:15:18.247774488Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-t8dkq-6vlv4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-t8dkq-6vlv4: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9d580\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-vrplznt5v54r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85d24adc-6f21-47c1-8812-f58f8001bb49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85d24adc-6f21-47c1-8812-f58f8001bb49\",\n      \"name\": \"85d24adc-6f21-47c1-8812-f58f8001bb49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:56:13.127412921Z\",\n      \"endTime\": \"2026-10-02T08:15:18.247774488Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-t8dkq-6vlv4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-t8dkq-6vlv4: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3add730a4d0f461956b699cfb69439560ad92350a3f3abae2844261da6b72dcf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"8ae482f04181781883adcf3137eae2256182da0fa9081ef6d8de86debd8ad010","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000666c60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bgj7nclctwxw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\",\n      \"name\": \"29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:52:01.290279416Z\",\n      \"endTime\": \"2026-10-02T09:11:09.436811147Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-4k649: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bgj7nclctwxw--managed/providers/Microsoft.Compute/virtualMachines/h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-4k649: Creating: virtualmachine creating or updating\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bgj7nclctwxw--managed/providers/Microsoft.Compute/virtualMachines/h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000666c60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bgj7nclctwxw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\",\n      \"name\": \"29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:52:01.290279416Z\",\n      \"endTime\": \"2026-10-02T09:11:09.436811147Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-4k649: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bgj7nclctwxw--managed/providers/Microsoft.Compute/virtualMachines/h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-4k649: Creating: virtualmachine creating or updating\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bgj7nclctwxw--managed/providers/Microsoft.Compute/virtualMachines/h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}},{"row_id":"434022071ee4c10aaca9bb6a368811b5ee3ecd1fb7e0735c94eb8f09b2cf6ba2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"fdc0e08684b498bbfb00d4bb852da4d3016c65a4ec84f977294ad7bdbf25c4a1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-q645ss\" for cluster \"cluster-candidate-4-23-q645ss\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00147edc0\u003e: \n    failed to create NodePool np-q645ss: failed waiting for nodepool=\"np-q645ss\" for cluster \"cluster-candidate-4-23-q645ss\" in resourcegroup=\"rg-candidate-4-23-97lbmpnc95ml\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\",\n      \"name\": \"e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:22:25.175276447Z\",\n      \"endTime\": \"2026-10-02T08:41:25.163899948Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-b9h5q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-rcd4l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-b9h5q: Creating: virtualmachine creating or updating\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-rcd4l: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-q645ss\" for cluster \"cluster-candidate-4-23-q645ss\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00147edc0\u003e: \n    failed to create NodePool np-q645ss: failed waiting for nodepool=\"np-q645ss\" for cluster \"cluster-candidate-4-23-q645ss\" in resourcegroup=\"rg-candidate-4-23-97lbmpnc95ml\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\",\n      \"name\": \"e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:22:25.175276447Z\",\n      \"endTime\": \"2026-10-02T08:41:25.163899948Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-b9h5q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-rcd4l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-b9h5q: Creating: virtualmachine creating or updating\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-rcd4l: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"45c195e887f303240824afdd140fc3d19d456ebc8956d20da34436bb61fb655d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"4bb202a926156d42d840b7a9818cb5b7935514c66bb61eb6995e5c5d73101ec2","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea0360\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5w2whf62mkf6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd107ed-298a-4008-9f25-801165f1c37e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd107ed-298a-4008-9f25-801165f1c37e\",\n      \"name\": \"ffd107ed-298a-4008-9f25-801165f1c37e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:56:52.717139839Z\",\n      \"endTime\": \"2026-10-02T08:15:58.568270167Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5w2whf62mkf6--managed/providers/Microsoft.Compute/virtualMachines/v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5w2whf62mkf6--managed/providers/Microsoft.Compute/virtualMachines/v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ea0360\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5w2whf62mkf6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd107ed-298a-4008-9f25-801165f1c37e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd107ed-298a-4008-9f25-801165f1c37e\",\n      \"name\": \"ffd107ed-298a-4008-9f25-801165f1c37e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:56:52.717139839Z\",\n      \"endTime\": \"2026-10-02T08:15:58.568270167Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5w2whf62mkf6--managed/providers/Microsoft.Compute/virtualMachines/v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5w2whf62mkf6--managed/providers/Microsoft.Compute/virtualMachines/v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"47d20a86aa1a33efb166b16b730c3bde80133830b4a727cf87b9e75916dd82e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"615a1a8b96dee5eb1bf6319d009a9ff736c86e855049cffe408293ba6acfef53","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kf44kx\" for cluster \"cluster-candidate-4-22-kf44kx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001670be0\u003e: \n    failed to create NodePool np-kf44kx: failed waiting for nodepool=\"np-kf44kx\" for cluster \"cluster-candidate-4-22-kf44kx\" in resourcegroup=\"rg-candidate-4-22-p972hm7v767t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f5cc41ef-2098-4080-80d0-52d4a6c605b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f5cc41ef-2098-4080-80d0-52d4a6c605b2\",\n      \"name\": \"f5cc41ef-2098-4080-80d0-52d4a6c605b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:24:18.844576711Z\",\n      \"endTime\": \"2026-10-02T08:43:25.00034882Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-ff2wq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-p7lf6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-ff2wq: Creating: virtualmachine creating or updating\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-p7lf6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kf44kx\" for cluster \"cluster-candidate-4-22-kf44kx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001670be0\u003e: \n    failed to create NodePool np-kf44kx: failed waiting for nodepool=\"np-kf44kx\" for cluster \"cluster-candidate-4-22-kf44kx\" in resourcegroup=\"rg-candidate-4-22-p972hm7v767t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f5cc41ef-2098-4080-80d0-52d4a6c605b2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f5cc41ef-2098-4080-80d0-52d4a6c605b2\",\n      \"name\": \"f5cc41ef-2098-4080-80d0-52d4a6c605b2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:24:18.844576711Z\",\n      \"endTime\": \"2026-10-02T08:43:25.00034882Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-ff2wq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-p7lf6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-ff2wq: Creating: virtualmachine creating or updating\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-p7lf6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"69668cb5819485ec95ab221da8e51cc7b89985fbaa7337ca405d514f206ca508","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"e764784e704e2e2b27d861fbb0e9e4e39307c4efc2358883f5a0a0bfaf1ece26","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qbwjq7\" for cluster \"cluster-candidate-5-1-qbwjq7\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f28640\u003e: \n    failed to create NodePool np-qbwjq7: failed waiting for nodepool=\"np-qbwjq7\" for cluster \"cluster-candidate-5-1-qbwjq7\" in resourcegroup=\"rg-candidate-5-1-cxt8gqtcm56h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/411adf6f-822a-4f98-bc7d-64851f61532c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/411adf6f-822a-4f98-bc7d-64851f61532c\",\n      \"name\": \"411adf6f-822a-4f98-bc7d-64851f61532c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:23:47.400908917Z\",\n      \"endTime\": \"2026-10-02T08:42:55.012095583Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-dhvsm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-zlljx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-dhvsm: Creating: virtualmachine creating or updating\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-zlljx: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qbwjq7\" for cluster \"cluster-candidate-5-1-qbwjq7\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f28640\u003e: \n    failed to create NodePool np-qbwjq7: failed waiting for nodepool=\"np-qbwjq7\" for cluster \"cluster-candidate-5-1-qbwjq7\" in resourcegroup=\"rg-candidate-5-1-cxt8gqtcm56h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/411adf6f-822a-4f98-bc7d-64851f61532c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/411adf6f-822a-4f98-bc7d-64851f61532c\",\n      \"name\": \"411adf6f-822a-4f98-bc7d-64851f61532c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:23:47.400908917Z\",\n      \"endTime\": \"2026-10-02T08:42:55.012095583Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-dhvsm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-zlljx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-dhvsm: Creating: virtualmachine creating or updating\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-zlljx: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"76bb947cd003eb4cd213e3dc50e7082655c5e22c2f3b96bd9622950d593acaf8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"d25ebb7feb22db43b9925f83b8d322fdcc4c54a5432fbc9ce5efb6aae6f0e97b","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e761b0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-dwl8n7k5d292\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n      \"name\": \"0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:58:36.615555909Z\",\n      \"endTime\": \"2026-10-02T08:17:38.532193005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e761b0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-dwl8n7k5d292\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n      \"name\": \"0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:58:36.615555909Z\",\n      \"endTime\": \"2026-10-02T08:17:38.532193005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"77c04b42c3abea3eeb95b33bf011bf1cf93dc9673e81f284e4d654d077fd6692","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"6c3e750da0e9592b1f2dec07e4441b5b99e79bcaca5632e733e0724a36755aaf","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008aece0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-plwp75qbhvcq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d68804b-aa1f-441d-9c44-b697ee5be0e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d68804b-aa1f-441d-9c44-b697ee5be0e1\",\n      \"name\": \"0d68804b-aa1f-441d-9c44-b697ee5be0e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:58:27.095320904Z\",\n      \"endTime\": \"2026-10-02T08:17:28.456252233Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-j6stv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-jcmz5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-j6stv: Creating: virtualmachine creating or updating\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-jcmz5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008aece0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-plwp75qbhvcq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d68804b-aa1f-441d-9c44-b697ee5be0e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d68804b-aa1f-441d-9c44-b697ee5be0e1\",\n      \"name\": \"0d68804b-aa1f-441d-9c44-b697ee5be0e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:58:27.095320904Z\",\n      \"endTime\": \"2026-10-02T08:17:28.456252233Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-j6stv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-jcmz5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-j6stv: Creating: virtualmachine creating or updating\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-jcmz5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9fd7b20df5e7be1c66779b9074e9d22fb23832a595df836266165ee8c1132e6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"04d1cb3cb4234161cc07dedd18e19e98c569ee86bfc6884e682791a6475e5f03","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-65j846\" for cluster \"cluster-candidate-4-21-65j846\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd8540\u003e: \n    failed to create NodePool np-65j846: failed waiting for nodepool=\"np-65j846\" for cluster \"cluster-candidate-4-21-65j846\" in resourcegroup=\"rg-candidate-4-21-8dj7spdjvcps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4b395e6-2b88-4e54-814b-bcbf478cdffc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4b395e6-2b88-4e54-814b-bcbf478cdffc\",\n      \"name\": \"d4b395e6-2b88-4e54-814b-bcbf478cdffc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:23:11.897579976Z\",\n      \"endTime\": \"2026-10-02T08:42:15.234899325Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-dczz6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-hp4tn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-dczz6: Creating: virtualmachine creating or updating\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-hp4tn: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-65j846\" for cluster \"cluster-candidate-4-21-65j846\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd8540\u003e: \n    failed to create NodePool np-65j846: failed waiting for nodepool=\"np-65j846\" for cluster \"cluster-candidate-4-21-65j846\" in resourcegroup=\"rg-candidate-4-21-8dj7spdjvcps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4b395e6-2b88-4e54-814b-bcbf478cdffc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4b395e6-2b88-4e54-814b-bcbf478cdffc\",\n      \"name\": \"d4b395e6-2b88-4e54-814b-bcbf478cdffc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:23:11.897579976Z\",\n      \"endTime\": \"2026-10-02T08:42:15.234899325Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-dczz6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-hp4tn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-dczz6: Creating: virtualmachine creating or updating\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-hp4tn: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"db267170cc53eb6b28a1be1aacea12ce6ef663f1fc63f93670e625ebf2b424a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"ee727effd5d95b42b1f21ecbaf21ff986de2285544c181405cc38338639d85a7","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002194198\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kkklwf69d6rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6596056-a431-4061-9fcc-922385eb88b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6596056-a431-4061-9fcc-922385eb88b7\",\n      \"name\": \"f6596056-a431-4061-9fcc-922385eb88b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:58:26.598790316Z\",\n      \"endTime\": \"2026-10-02T08:17:28.2532697Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-t9npj-5lpcd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-t9npj-z8x9w: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-t9npj-5lpcd: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-t9npj-z8x9w: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002194198\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kkklwf69d6rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6596056-a431-4061-9fcc-922385eb88b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6596056-a431-4061-9fcc-922385eb88b7\",\n      \"name\": \"f6596056-a431-4061-9fcc-922385eb88b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:58:26.598790316Z\",\n      \"endTime\": \"2026-10-02T08:17:28.2532697Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-t9npj-5lpcd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-t9npj-z8x9w: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-t9npj-5lpcd: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-t9npj-z8x9w: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dfd01cc5b407694b7f2d98a60e32845a0d41aa8f4609e7d7cecb9c2939272dbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"aa9cf2d4709f42c9eb5dad95a56141738554f1b7d752de0d4b0ca863ed35f08d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-96gwgz\" for cluster \"cluster-candidate-5-0-96gwgz\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cf2260\u003e: \n    failed to create NodePool np-96gwgz: failed waiting for nodepool=\"np-96gwgz\" for cluster \"cluster-candidate-5-0-96gwgz\" in resourcegroup=\"rg-candidate-5-0-84kwhzj8r7w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7df18c5-1f74-4f58-93d8-951ddd66b332\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7df18c5-1f74-4f58-93d8-951ddd66b332\",\n      \"name\": \"c7df18c5-1f74-4f58-93d8-951ddd66b332\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:23:19.198498666Z\",\n      \"endTime\": \"2026-10-02T08:42:25.18491611Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-4frj5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-d5fmk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-4frj5: Creating: virtualmachine creating or updating\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-d5fmk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-96gwgz\" for cluster \"cluster-candidate-5-0-96gwgz\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cf2260\u003e: \n    failed to create NodePool np-96gwgz: failed waiting for nodepool=\"np-96gwgz\" for cluster \"cluster-candidate-5-0-96gwgz\" in resourcegroup=\"rg-candidate-5-0-84kwhzj8r7w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7df18c5-1f74-4f58-93d8-951ddd66b332\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7df18c5-1f74-4f58-93d8-951ddd66b332\",\n      \"name\": \"c7df18c5-1f74-4f58-93d8-951ddd66b332\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:23:19.198498666Z\",\n      \"endTime\": \"2026-10-02T08:42:25.18491611Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-4frj5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-d5fmk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-4frj5: Creating: virtualmachine creating or updating\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-d5fmk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f9e478368f37ecd0b1347af680ab6aa57f344170bbde5a159c6b1b05b5e2e239","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"de872c4f9bed78e595aa0309be8b2145ae00fa864cdae74c62b196f02e7a7c68","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tgr2wf\" for cluster \"cluster-candidate-4-20-tgr2wf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e1d940\u003e: \n    failed to create NodePool np-tgr2wf: failed waiting for nodepool=\"np-tgr2wf\" for cluster \"cluster-candidate-4-20-tgr2wf\" in resourcegroup=\"rg-candidate-4-20-r2lrfp2tbsqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0734784e-738a-4b64-ad9c-e373ea6e9682\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0734784e-738a-4b64-ad9c-e373ea6e9682\",\n      \"name\": \"0734784e-738a-4b64-ad9c-e373ea6e9682\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:23:20.483388473Z\",\n      \"endTime\": \"2026-10-02T08:42:25.166235902Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-jmm8m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-k7sxl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-jmm8m: Creating: virtualmachine creating or updating\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-k7sxl: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tgr2wf\" for cluster \"cluster-candidate-4-20-tgr2wf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e1d940\u003e: \n    failed to create NodePool np-tgr2wf: failed waiting for nodepool=\"np-tgr2wf\" for cluster \"cluster-candidate-4-20-tgr2wf\" in resourcegroup=\"rg-candidate-4-20-r2lrfp2tbsqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0734784e-738a-4b64-ad9c-e373ea6e9682\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0734784e-738a-4b64-ad9c-e373ea6e9682\",\n      \"name\": \"0734784e-738a-4b64-ad9c-e373ea6e9682\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T08:23:20.483388473Z\",\n      \"endTime\": \"2026-10-02T08:42:25.166235902Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-jmm8m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-k7sxl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-jmm8m: Creating: virtualmachine creating or updating\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-k7sxl: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T02:00:29Z","started_at":"2026-10-02T02:00:29Z","completed_at":"2026-10-02T05:11:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T05:27:48.820525766Z","timing_metadata_checked_at":"2026-10-02T05:27:48.820525766Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T05:27:48.823897156Z","region_metadata_checked_at":"2026-10-02T05:27:48.823897156Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"07b4e87bab48a73502f78d5af689f69fe1f8ed7ff9f6b86c2e522a5a4c796d20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"8712867601ac02520c8073037f11212160d4494ca873f5f46983b954dae750d9","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001276700\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d53fa45c-a9c4-43a3-b87a-c16df786eeb4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d53fa45c-a9c4-43a3-b87a-c16df786eeb4\",\n      \"name\": \"d53fa45c-a9c4-43a3-b87a-c16df786eeb4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T04:09:42.171003862Z\",\n      \"endTime\": \"2026-10-02T04:28:46.827990048Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-n4l4p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-qtmzk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-n4l4p: Creating: virtualmachine creating or updating\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-qtmzk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001276700\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d53fa45c-a9c4-43a3-b87a-c16df786eeb4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d53fa45c-a9c4-43a3-b87a-c16df786eeb4\",\n      \"name\": \"d53fa45c-a9c4-43a3-b87a-c16df786eeb4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T04:09:42.171003862Z\",\n      \"endTime\": \"2026-10-02T04:28:46.827990048Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-n4l4p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-qtmzk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-n4l4p: Creating: virtualmachine creating or updating\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-qtmzk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}},{"row_id":"1fae92c6e879f21af0a90c9b3f9e558595dc88652f8c5c2c8eb4359cc930a059","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"5976bd54ef1912d8a56fae856cd640929ca801e0b38df661fe97065f1e1ddd1e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-9rfwql\" for cluster \"cluster-candidate-5-1-9rfwql\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001103040\u003e: \n    failed to create NodePool np-9rfwql: failed waiting for nodepool=\"np-9rfwql\" for cluster \"cluster-candidate-5-1-9rfwql\" in resourcegroup=\"rg-candidate-5-1-jznzltj2rxxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fd61fa-5104-45ad-bf92-0d32ab5a44e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fd61fa-5104-45ad-bf92-0d32ab5a44e5\",\n      \"name\": \"34fd61fa-5104-45ad-bf92-0d32ab5a44e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:44:17.773093204Z\",\n      \"endTime\": \"2026-10-02T04:03:23.679415683Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-9ptnl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-gscjf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-9ptnl: Creating: virtualmachine creating or updating\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-gscjf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-9rfwql\" for cluster \"cluster-candidate-5-1-9rfwql\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001103040\u003e: \n    failed to create NodePool np-9rfwql: failed waiting for nodepool=\"np-9rfwql\" for cluster \"cluster-candidate-5-1-9rfwql\" in resourcegroup=\"rg-candidate-5-1-jznzltj2rxxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fd61fa-5104-45ad-bf92-0d32ab5a44e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fd61fa-5104-45ad-bf92-0d32ab5a44e5\",\n      \"name\": \"34fd61fa-5104-45ad-bf92-0d32ab5a44e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:44:17.773093204Z\",\n      \"endTime\": \"2026-10-02T04:03:23.679415683Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-9ptnl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-gscjf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-9ptnl: Creating: virtualmachine creating or updating\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-gscjf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"22f9c91ba521286f1112399191933a96daafb6f31cd806329d9f52862c0547cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"91fd4fecbd77ed6f997de413f52e960fcad2e588e3fa8f18e505fb83e8c11564","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002984078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-4httpsjkvppc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/27c9b728-c34e-4afa-93f4-b5118810ce82\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/27c9b728-c34e-4afa-93f4-b5118810ce82\",\n      \"name\": \"27c9b728-c34e-4afa-93f4-b5118810ce82\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:19:57.766414352Z\",\n      \"endTime\": \"2026-10-02T03:39:03.235272547Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-lszrc-8znc6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-lszrc-nxxm5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-lszrc-8znc6: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-lszrc-nxxm5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002984078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-4httpsjkvppc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/27c9b728-c34e-4afa-93f4-b5118810ce82\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/27c9b728-c34e-4afa-93f4-b5118810ce82\",\n      \"name\": \"27c9b728-c34e-4afa-93f4-b5118810ce82\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:19:57.766414352Z\",\n      \"endTime\": \"2026-10-02T03:39:03.235272547Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-lszrc-8znc6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-lszrc-nxxm5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-lszrc-8znc6: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-lszrc-nxxm5: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"445c691fc32c05ab1b8738556fb7f1484753f873c095e2cfd4b6c6772517b5d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"297f54696f0c10752edca7d9cf5c5d2f98707884ddccf8ef2f952e32d9f597c8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fs2tl4\" for cluster \"cluster-candidate-4-22-fs2tl4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dde140\u003e: \n    failed to create NodePool np-fs2tl4: failed waiting for nodepool=\"np-fs2tl4\" for cluster \"cluster-candidate-4-22-fs2tl4\" in resourcegroup=\"rg-candidate-4-22-jzw6chr4d7m4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/af47af07-70e2-47bd-9306-0c7f478d186e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/af47af07-70e2-47bd-9306-0c7f478d186e\",\n      \"name\": \"af47af07-70e2-47bd-9306-0c7f478d186e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:39:14.039068534Z\",\n      \"endTime\": \"2026-10-02T03:58:23.505666093Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-hwrpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-mh58m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-hwrpc: Creating: virtualmachine creating or updating\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-mh58m: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fs2tl4\" for cluster \"cluster-candidate-4-22-fs2tl4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dde140\u003e: \n    failed to create NodePool np-fs2tl4: failed waiting for nodepool=\"np-fs2tl4\" for cluster \"cluster-candidate-4-22-fs2tl4\" in resourcegroup=\"rg-candidate-4-22-jzw6chr4d7m4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/af47af07-70e2-47bd-9306-0c7f478d186e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/af47af07-70e2-47bd-9306-0c7f478d186e\",\n      \"name\": \"af47af07-70e2-47bd-9306-0c7f478d186e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:39:14.039068534Z\",\n      \"endTime\": \"2026-10-02T03:58:23.505666093Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-hwrpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-mh58m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-hwrpc: Creating: virtualmachine creating or updating\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-mh58m: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6ef4b5578883c7582d166104a0a47c734f8dadf2a4ee097a0b0af9768ff6f4e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"7c052020cd8edffb258e72da7eba5bdb0cf97b0d362af401b3cb610f8b906cf6","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000943f60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6sjr57crkxhh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a2572ed-7c03-46c2-98bb-89a19d5f777a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a2572ed-7c03-46c2-98bb-89a19d5f777a\",\n      \"name\": \"3a2572ed-7c03-46c2-98bb-89a19d5f777a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:19:47.10992898Z\",\n      \"endTime\": \"2026-10-02T03:38:53.263922354Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-9k9hg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-fjdfs: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-9k9hg: Creating: virtualmachine creating or updating\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-fjdfs: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000943f60\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6sjr57crkxhh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a2572ed-7c03-46c2-98bb-89a19d5f777a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a2572ed-7c03-46c2-98bb-89a19d5f777a\",\n      \"name\": \"3a2572ed-7c03-46c2-98bb-89a19d5f777a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:19:47.10992898Z\",\n      \"endTime\": \"2026-10-02T03:38:53.263922354Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-9k9hg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-fjdfs: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-9k9hg: Creating: virtualmachine creating or updating\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-fjdfs: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"72b0b59f816116f4fff569918df1af22c32e84bf1965b5228e7e015a6471ba06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"8750a839c279cdf29c1a82ccb6189d7fb26d4cb33d0d9e3708556a84f521bd7e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2rhqzm\" for cluster \"cluster-candidate-4-23-2rhqzm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90900\u003e: \n    failed to create NodePool np-2rhqzm: failed waiting for nodepool=\"np-2rhqzm\" for cluster \"cluster-candidate-4-23-2rhqzm\" in resourcegroup=\"rg-candidate-4-23-hvftzsslsv5w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83438069-4881-4a7c-99c3-04167cb9898c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83438069-4881-4a7c-99c3-04167cb9898c\",\n      \"name\": \"83438069-4881-4a7c-99c3-04167cb9898c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:40:49.053626001Z\",\n      \"endTime\": \"2026-10-02T03:59:53.321378994Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-jdv5b: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-r825r: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-jdv5b: Creating: virtualmachine creating or updating\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-r825r: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2rhqzm\" for cluster \"cluster-candidate-4-23-2rhqzm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90900\u003e: \n    failed to create NodePool np-2rhqzm: failed waiting for nodepool=\"np-2rhqzm\" for cluster \"cluster-candidate-4-23-2rhqzm\" in resourcegroup=\"rg-candidate-4-23-hvftzsslsv5w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83438069-4881-4a7c-99c3-04167cb9898c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83438069-4881-4a7c-99c3-04167cb9898c\",\n      \"name\": \"83438069-4881-4a7c-99c3-04167cb9898c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:40:49.053626001Z\",\n      \"endTime\": \"2026-10-02T03:59:53.321378994Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-jdv5b: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-r825r: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-jdv5b: Creating: virtualmachine creating or updating\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-r825r: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"74494e992ed4abcc34c99899efa3343e2c68f1c01682ea9b7400f4718bc995e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"9cbcd1fcc44b9dcfeac71a715da83e6c8d7bf9ceee862c10b305cfe44d73e05f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lfrdtx\" for cluster \"cluster-candidate-4-21-lfrdtx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a86ae0\u003e: \n    failed to create NodePool np-lfrdtx: failed waiting for nodepool=\"np-lfrdtx\" for cluster \"cluster-candidate-4-21-lfrdtx\" in resourcegroup=\"rg-candidate-4-21-v7j8q5wv69rf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8487946-6d32-4b69-bbc8-793ffc91ce5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8487946-6d32-4b69-bbc8-793ffc91ce5f\",\n      \"name\": \"a8487946-6d32-4b69-bbc8-793ffc91ce5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:40:32.227275462Z\",\n      \"endTime\": \"2026-10-02T03:59:33.308103816Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-58v7d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-5gbq8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-58v7d: Creating: virtualmachine creating or updating\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-5gbq8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lfrdtx\" for cluster \"cluster-candidate-4-21-lfrdtx\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a86ae0\u003e: \n    failed to create NodePool np-lfrdtx: failed waiting for nodepool=\"np-lfrdtx\" for cluster \"cluster-candidate-4-21-lfrdtx\" in resourcegroup=\"rg-candidate-4-21-v7j8q5wv69rf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8487946-6d32-4b69-bbc8-793ffc91ce5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8487946-6d32-4b69-bbc8-793ffc91ce5f\",\n      \"name\": \"a8487946-6d32-4b69-bbc8-793ffc91ce5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:40:32.227275462Z\",\n      \"endTime\": \"2026-10-02T03:59:33.308103816Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-58v7d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-5gbq8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-58v7d: Creating: virtualmachine creating or updating\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-5gbq8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"80cf20a9c9f509bba5e62e463711a1e1c06212f7805ee16146090e82e7566897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"9011219d09cdf07f6d086e4ba313338a0c41037e23acedb902f10a2f46cf7b4c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-s4kd9s\" for cluster \"cluster-candidate-4-20-s4kd9s\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0b4a0\u003e: \n    failed to create NodePool np-s4kd9s: failed waiting for nodepool=\"np-s4kd9s\" for cluster \"cluster-candidate-4-20-s4kd9s\" in resourcegroup=\"rg-candidate-4-20-zlm28jv4jtcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\",\n      \"name\": \"7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:38:55.251415601Z\",\n      \"endTime\": \"2026-10-02T03:58:03.48228814Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-mn8wj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-st4n7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-mn8wj: Creating: virtualmachine creating or updating\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-st4n7: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-s4kd9s\" for cluster \"cluster-candidate-4-20-s4kd9s\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0b4a0\u003e: \n    failed to create NodePool np-s4kd9s: failed waiting for nodepool=\"np-s4kd9s\" for cluster \"cluster-candidate-4-20-s4kd9s\" in resourcegroup=\"rg-candidate-4-20-zlm28jv4jtcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\",\n      \"name\": \"7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:38:55.251415601Z\",\n      \"endTime\": \"2026-10-02T03:58:03.48228814Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-mn8wj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-st4n7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-mn8wj: Creating: virtualmachine creating or updating\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-st4n7: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"994cb16080650a54048b98abc269c5ac388e88b8448457d0a7057e29e73b6ca5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"4bfb160db3532883f03c196af07faf62b876ad474cc0c49e5e501e7fd65537d8","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00064c1e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-gs752pln4txj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\",\n      \"name\": \"9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:18:04.846590038Z\",\n      \"endTime\": \"2026-10-02T03:37:13.231851998Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-5dbrz-khxf4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-5dbrz-khxf4: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00064c1e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-gs752pln4txj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\",\n      \"name\": \"9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:18:04.846590038Z\",\n      \"endTime\": \"2026-10-02T03:37:13.231851998Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-5dbrz-khxf4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-5dbrz-khxf4: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ad04048961b5f0b481e79c6e7a4ca829f00bffd14496dc3ff9939ace3fce0cd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"74c4a0991a9347ceeb2b3ab53adc95cedd09f120cf556321c9b74ae4d3616c90","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0011d4048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rrrmbd9jmmdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d52358f-8309-4bc7-af9f-a5884f9307e0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d52358f-8309-4bc7-af9f-a5884f9307e0\",\n      \"name\": \"6d52358f-8309-4bc7-af9f-a5884f9307e0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:16:58.467567062Z\",\n      \"endTime\": \"2026-10-02T03:36:03.226032745Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-sqrfn-pjdbq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-sqrfn-vtbbr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-sqrfn-pjdbq: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-sqrfn-vtbbr: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0011d4048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rrrmbd9jmmdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d52358f-8309-4bc7-af9f-a5884f9307e0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d52358f-8309-4bc7-af9f-a5884f9307e0\",\n      \"name\": \"6d52358f-8309-4bc7-af9f-a5884f9307e0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:16:58.467567062Z\",\n      \"endTime\": \"2026-10-02T03:36:03.226032745Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-sqrfn-pjdbq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-sqrfn-vtbbr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-sqrfn-pjdbq: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-sqrfn-vtbbr: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f3edaed51a0f30a57e00e2785d6ce36edebf24c4d9620d26d2389642b7c98f70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"299db9852cb6ec237c0f1bb6b3917d59644cef430e83fc2d28e8943682b7e108","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2pqt8t\" for cluster \"cluster-candidate-5-0-2pqt8t\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e6c920\u003e: \n    failed to create NodePool np-2pqt8t: failed waiting for nodepool=\"np-2pqt8t\" for cluster \"cluster-candidate-5-0-2pqt8t\" in resourcegroup=\"rg-candidate-5-0-5cmhl28njm2n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1671f2aa-76df-4172-89e0-297ac668f021\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1671f2aa-76df-4172-89e0-297ac668f021\",\n      \"name\": \"1671f2aa-76df-4172-89e0-297ac668f021\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:42:21.546109728Z\",\n      \"endTime\": \"2026-10-02T04:01:23.351194166Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-8bg9m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-v5mdw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-8bg9m: Creating: virtualmachine creating or updating\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-v5mdw: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2pqt8t\" for cluster \"cluster-candidate-5-0-2pqt8t\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e6c920\u003e: \n    failed to create NodePool np-2pqt8t: failed waiting for nodepool=\"np-2pqt8t\" for cluster \"cluster-candidate-5-0-2pqt8t\" in resourcegroup=\"rg-candidate-5-0-5cmhl28njm2n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1671f2aa-76df-4172-89e0-297ac668f021\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1671f2aa-76df-4172-89e0-297ac668f021\",\n      \"name\": \"1671f2aa-76df-4172-89e0-297ac668f021\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T03:42:21.546109728Z\",\n      \"endTime\": \"2026-10-02T04:01:23.351194166Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-8bg9m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-v5mdw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-8bg9m: Creating: virtualmachine creating or updating\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-v5mdw: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T00:53:40Z","started_at":"2026-10-02T00:53:40Z","completed_at":"2026-10-02T02:59:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T03:02:56.086682389Z","timing_metadata_checked_at":"2026-10-02T03:02:56.086682389Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T03:02:56.093669068Z","region_metadata_checked_at":"2026-10-02T03:02:56.093669068Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"3b16639bc16dbbe14f5810b05c126b6bd3984226ddf176e8d37bf1be9f2488f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"6d948816532f93e1d1f025d9ed2447096861bf76e34447e5173e22879b9214cf","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002434018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9gqhrxhspmxj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2689a9-2918-4b62-b602-77629a36d12f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2689a9-2918-4b62-b602-77629a36d12f\",\n      \"name\": \"1e2689a9-2918-4b62-b602-77629a36d12f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:12:51.230795814Z\",\n      \"endTime\": \"2026-10-02T01:31:55.658354454Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-zw5cd-7wmwm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-zw5cd-t5w4x: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-zw5cd-7wmwm: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-zw5cd-t5w4x: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002434018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9gqhrxhspmxj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2689a9-2918-4b62-b602-77629a36d12f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2689a9-2918-4b62-b602-77629a36d12f\",\n      \"name\": \"1e2689a9-2918-4b62-b602-77629a36d12f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:12:51.230795814Z\",\n      \"endTime\": \"2026-10-02T01:31:55.658354454Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-zw5cd-7wmwm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-zw5cd-t5w4x: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-zw5cd-7wmwm: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-zw5cd-t5w4x: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6b4c95a67adaafe3b7d63c9fd89bab8a39011d0e1b1e5c8dba06b1307fe4076d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"9ab2b1eb14ea74c42017c9099045c52a9eb9611a341b4d1c5df66b6729fb582b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tgw4bw\" for cluster \"cluster-candidate-5-0-tgw4bw\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013d7020\u003e: \n    failed to create NodePool np-tgw4bw: failed waiting for nodepool=\"np-tgw4bw\" for cluster \"cluster-candidate-5-0-tgw4bw\" in resourcegroup=\"rg-candidate-5-0-c5dplrlznljs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4804927-bde6-4d56-b2a2-861fb62752d1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4804927-bde6-4d56-b2a2-861fb62752d1\",\n      \"name\": \"e4804927-bde6-4d56-b2a2-861fb62752d1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:36:37.66754756Z\",\n      \"endTime\": \"2026-10-02T01:55:45.209455595Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-h9x7v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-q4chj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-h9x7v: Creating: virtualmachine creating or updating\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-q4chj: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tgw4bw\" for cluster \"cluster-candidate-5-0-tgw4bw\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013d7020\u003e: \n    failed to create NodePool np-tgw4bw: failed waiting for nodepool=\"np-tgw4bw\" for cluster \"cluster-candidate-5-0-tgw4bw\" in resourcegroup=\"rg-candidate-5-0-c5dplrlznljs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4804927-bde6-4d56-b2a2-861fb62752d1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4804927-bde6-4d56-b2a2-861fb62752d1\",\n      \"name\": \"e4804927-bde6-4d56-b2a2-861fb62752d1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:36:37.66754756Z\",\n      \"endTime\": \"2026-10-02T01:55:45.209455595Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-h9x7v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-q4chj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-h9x7v: Creating: virtualmachine creating or updating\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-q4chj: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6cfe946d5f677115c5c8031bc667f1dcea84285080095415791b7620da122103","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"a0aef2ff3133e203b8b6be69e707c3391a26d7478459a0d095a7d80d52b7f77e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101d200\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-2lqkfn8tvwwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4098a97-bd38-463d-bee6-54c33fcd21ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4098a97-bd38-463d-bee6-54c33fcd21ea\",\n      \"name\": \"f4098a97-bd38-463d-bee6-54c33fcd21ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:12:48.719133172Z\",\n      \"endTime\": \"2026-10-02T01:31:55.837797735Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-kdjwf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-q7z6x: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-kdjwf: Creating: virtualmachine creating or updating\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-q7z6x: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00101d200\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-2lqkfn8tvwwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4098a97-bd38-463d-bee6-54c33fcd21ea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4098a97-bd38-463d-bee6-54c33fcd21ea\",\n      \"name\": \"f4098a97-bd38-463d-bee6-54c33fcd21ea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:12:48.719133172Z\",\n      \"endTime\": \"2026-10-02T01:31:55.837797735Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-kdjwf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-q7z6x: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-kdjwf: Creating: virtualmachine creating or updating\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-q7z6x: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6ecd9b8fd182cba134a98a738b7a020e73d790f531ab5de37cd2bbc7e510a7fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"2e01f42474260e70f5cc8017801f4c45f4df9dbc895992ce253206654bfb0587","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2dtcq8\" for cluster \"cluster-candidate-4-20-2dtcq8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018500e0\u003e: \n    failed to create NodePool np-2dtcq8: failed waiting for nodepool=\"np-2dtcq8\" for cluster \"cluster-candidate-4-20-2dtcq8\" in resourcegroup=\"rg-candidate-4-20-wsgq7rfg4j9c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\",\n      \"name\": \"4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:35:03.016964221Z\",\n      \"endTime\": \"2026-10-02T01:54:05.336017308Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2dtcq8\" for cluster \"cluster-candidate-4-20-2dtcq8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0018500e0\u003e: \n    failed to create NodePool np-2dtcq8: failed waiting for nodepool=\"np-2dtcq8\" for cluster \"cluster-candidate-4-20-2dtcq8\" in resourcegroup=\"rg-candidate-4-20-wsgq7rfg4j9c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\",\n      \"name\": \"4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:35:03.016964221Z\",\n      \"endTime\": \"2026-10-02T01:54:05.336017308Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6ee29ba6ce642873f3ae4081c2b12ae165b5e71be7507b6512a426336aaa89af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"2c5ca9fee600aa40b6dd99a68079eed31e31c0041cca6c9bb81c4e2c28989095","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-z9vgpc\" for cluster \"cluster-candidate-4-23-z9vgpc\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d433c0\u003e: \n    failed to create NodePool np-z9vgpc: failed waiting for nodepool=\"np-z9vgpc\" for cluster \"cluster-candidate-4-23-z9vgpc\" in resourcegroup=\"rg-candidate-4-23-k2q8jrl7bhxd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5150081-901b-4236-a201-530c0a2b188a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5150081-901b-4236-a201-530c0a2b188a\",\n      \"name\": \"a5150081-901b-4236-a201-530c0a2b188a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:35:33.904818612Z\",\n      \"endTime\": \"2026-10-02T01:54:35.210883513Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-54zr9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-tfmvd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-54zr9: Creating: virtualmachine creating or updating\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-tfmvd: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-z9vgpc\" for cluster \"cluster-candidate-4-23-z9vgpc\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d433c0\u003e: \n    failed to create NodePool np-z9vgpc: failed waiting for nodepool=\"np-z9vgpc\" for cluster \"cluster-candidate-4-23-z9vgpc\" in resourcegroup=\"rg-candidate-4-23-k2q8jrl7bhxd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5150081-901b-4236-a201-530c0a2b188a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5150081-901b-4236-a201-530c0a2b188a\",\n      \"name\": \"a5150081-901b-4236-a201-530c0a2b188a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:35:33.904818612Z\",\n      \"endTime\": \"2026-10-02T01:54:35.210883513Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-54zr9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-tfmvd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-54zr9: Creating: virtualmachine creating or updating\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-tfmvd: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8884e88d6d84da2e7b9c0e73edf64fa7bf0b1182e9b1d3a8603c60c3f93abfec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"02f2d68ee929abf0d14dbf066cea47b45e31fa5a7861dd29f5ff1ae2dda13550","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002d92dc0\u003e: \n    failed to create NodePool np-ljll7c: failed waiting for nodepool=\"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\" in resourcegroup=\"rg-candidate-5-1-m96hr99msv6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\",\n      \"name\": \"3472529a-993b-497a-a853-58314ee41fea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:39:28.8432233Z\",\n      \"endTime\": \"2026-10-02T01:58:35.211672532Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002d92dc0\u003e: \n    failed to create NodePool np-ljll7c: failed waiting for nodepool=\"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\" in resourcegroup=\"rg-candidate-5-1-m96hr99msv6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\",\n      \"name\": \"3472529a-993b-497a-a853-58314ee41fea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:39:28.8432233Z\",\n      \"endTime\": \"2026-10-02T01:58:35.211672532Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9035c099feda9d5c8c3f53f6e9ceba973d7e11554bdb29525b4d3635fa47cd50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"0d8fad05bfc91d3bf4587ff521f51f71dec38c4bd6167c681739b8c407410634","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b95a80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-vp5rgzktgrqp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7950f938-55ee-4ce6-a5ca-480917c685bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7950f938-55ee-4ce6-a5ca-480917c685bf\",\n      \"name\": \"7950f938-55ee-4ce6-a5ca-480917c685bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:15:07.199174306Z\",\n      \"endTime\": \"2026-10-02T01:34:15.817819834Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-vqbnm-zdcgw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-vqbnm-zdcgw: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b95a80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-vp5rgzktgrqp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7950f938-55ee-4ce6-a5ca-480917c685bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7950f938-55ee-4ce6-a5ca-480917c685bf\",\n      \"name\": \"7950f938-55ee-4ce6-a5ca-480917c685bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:15:07.199174306Z\",\n      \"endTime\": \"2026-10-02T01:34:15.817819834Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-vqbnm-zdcgw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-vqbnm-zdcgw: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c074fab4310135b3ac27c3fbc92a279b46d1962f358ad5fdfd4e9a262d6e651b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"556da369df5fa371fae5a268e2663151c01997d0b1338ed3e1e0b97061c14199","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pdsktt\" for cluster \"cluster-candidate-4-22-pdsktt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010d83a0\u003e: \n    failed to create NodePool np-pdsktt: failed waiting for nodepool=\"np-pdsktt\" for cluster \"cluster-candidate-4-22-pdsktt\" in resourcegroup=\"rg-candidate-4-22-5hpdwnfcpnnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/987287c4-95cd-422d-be6d-282705a06618\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/987287c4-95cd-422d-be6d-282705a06618\",\n      \"name\": \"987287c4-95cd-422d-be6d-282705a06618\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:35:11.708229062Z\",\n      \"endTime\": \"2026-10-02T01:54:15.19502959Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-b9m75: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-vsxkb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-b9m75: Creating: virtualmachine creating or updating\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-vsxkb: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pdsktt\" for cluster \"cluster-candidate-4-22-pdsktt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010d83a0\u003e: \n    failed to create NodePool np-pdsktt: failed waiting for nodepool=\"np-pdsktt\" for cluster \"cluster-candidate-4-22-pdsktt\" in resourcegroup=\"rg-candidate-4-22-5hpdwnfcpnnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/987287c4-95cd-422d-be6d-282705a06618\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/987287c4-95cd-422d-be6d-282705a06618\",\n      \"name\": \"987287c4-95cd-422d-be6d-282705a06618\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:35:11.708229062Z\",\n      \"endTime\": \"2026-10-02T01:54:15.19502959Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-b9m75: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-vsxkb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-b9m75: Creating: virtualmachine creating or updating\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-vsxkb: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ccdac94ec54475c82fab76ae46c0faddb036dd10e1f61328d8867d84ed81d02a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"fcbce16b0ec7dd896386a692502ebe83bdec088d3b3acd3226882a8945ae4909","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a9c060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-v2ffkzvzwhp9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/59c86be8-df72-4988-a51b-6b1b20b915f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/59c86be8-df72-4988-a51b-6b1b20b915f4\",\n      \"name\": \"59c86be8-df72-4988-a51b-6b1b20b915f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:13:42.494621442Z\",\n      \"endTime\": \"2026-10-02T01:32:45.646902114Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-69vhp-6nxhl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-69vhp-rch98: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-69vhp-6nxhl: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-69vhp-rch98: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a9c060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-v2ffkzvzwhp9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/59c86be8-df72-4988-a51b-6b1b20b915f4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/59c86be8-df72-4988-a51b-6b1b20b915f4\",\n      \"name\": \"59c86be8-df72-4988-a51b-6b1b20b915f4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:13:42.494621442Z\",\n      \"endTime\": \"2026-10-02T01:32:45.646902114Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-69vhp-6nxhl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-69vhp-rch98: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-69vhp-6nxhl: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-69vhp-rch98: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"dbe220fd3c4eaa185ec9ed8726b0f7b0e126dc229bf9a0899c82093132d36286","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"5a5ab12ceb7567e88b54656617d4a4e3c761b84d7c19fe667549b41a60c4797b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tnjk89\" for cluster \"cluster-candidate-4-21-tnjk89\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a24560\u003e: \n    failed to create NodePool np-tnjk89: failed waiting for nodepool=\"np-tnjk89\" for cluster \"cluster-candidate-4-21-tnjk89\" in resourcegroup=\"rg-candidate-4-21-hbwb4qtqm5m5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c80392bf-1395-461b-9b36-b3ca3dfa3732\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c80392bf-1395-461b-9b36-b3ca3dfa3732\",\n      \"name\": \"c80392bf-1395-461b-9b36-b3ca3dfa3732\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:34:52.67307417Z\",\n      \"endTime\": \"2026-10-02T01:53:55.817268557Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-mfsnf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-rmqgp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-mfsnf: Creating: virtualmachine creating or updating\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-rmqgp: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tnjk89\" for cluster \"cluster-candidate-4-21-tnjk89\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a24560\u003e: \n    failed to create NodePool np-tnjk89: failed waiting for nodepool=\"np-tnjk89\" for cluster \"cluster-candidate-4-21-tnjk89\" in resourcegroup=\"rg-candidate-4-21-hbwb4qtqm5m5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c80392bf-1395-461b-9b36-b3ca3dfa3732\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c80392bf-1395-461b-9b36-b3ca3dfa3732\",\n      \"name\": \"c80392bf-1395-461b-9b36-b3ca3dfa3732\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:34:52.67307417Z\",\n      \"endTime\": \"2026-10-02T01:53:55.817268557Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-mfsnf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-rmqgp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-mfsnf: Creating: virtualmachine creating or updating\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-rmqgp: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e8b6e5f980b89a3c447fa7c04b3863a137448f36c1a35ce9db2eb93a64f16e36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"4dcb3e82c312b27beb2846bcb4e3feac7ef1c6c538ba9c325c9e8ba1cd984500","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000eaa180\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-ks2bs7m442f9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3917dbfd-2f9f-4a35-8346-d9c0b133e313\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3917dbfd-2f9f-4a35-8346-d9c0b133e313\",\n      \"name\": \"3917dbfd-2f9f-4a35-8346-d9c0b133e313\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T02:04:51.953995015Z\",\n      \"endTime\": \"2026-10-02T02:23:58.43798257Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-85c4g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-g4qwf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-85c4g: Creating: virtualmachine creating or updating\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-g4qwf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000eaa180\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-ks2bs7m442f9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3917dbfd-2f9f-4a35-8346-d9c0b133e313\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3917dbfd-2f9f-4a35-8346-d9c0b133e313\",\n      \"name\": \"3917dbfd-2f9f-4a35-8346-d9c0b133e313\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T02:04:51.953995015Z\",\n      \"endTime\": \"2026-10-02T02:23:58.43798257Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-85c4g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-g4qwf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-85c4g: Creating: virtualmachine creating or updating\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-g4qwf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}