{"meta":{"date":"2026-10-08","timezone":"UTC","generated_at":"2026-10-30T17:07:14Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":7,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":5,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T23:00:38Z","started_at":"2026-10-08T23:00:38Z","completed_at":"2026-10-09T00:48:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T00:59:36.221154786Z","timing_metadata_checked_at":"2026-10-09T00:59:36.221154786Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T00:59:36.224861276Z","region_metadata_checked_at":"2026-10-09T00:59:36.224861276Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"206d5abc154cdb0ba32661e26002d4ca954ba241fb0d74fd519f149748b75aa8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"fc06ea34826977c15833fec7d2cf3a0aa68559a73c0a34b4f46fe7992c958855","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1bc3bcaaef00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8cplvshb5svv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1bc3bcaaef00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-8cplvshb5svv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4dd8639353e2e29ca870cbf053e369705a441d2dee2631e28520f138327aef81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"583203b05397b372a4354f60b57a0bc62bc0edfa59d540e09c922f53a5450367","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x17466b892d50\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-n8q8j5xgxdk4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x17466b892d50\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-n8q8j5xgxdk4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6927a42ae38084140fc26f8cf96613c6437635b5c206d211016b481a9cc39079","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"948425aa0583e35245acd6dfa26e29e48e09e7387b7c14f662b8e630e477a7b4","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2e62773595c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-m97nzsnh6v7k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2e62773595c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-m97nzsnh6v7k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6a5ffb775a61db9199bbcc0e043e00fefa37c972db5b1c1de62a03114ed96b50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"538bc1de69b7243ea254b7eafc69d03d17567c9e64b692cd74fb0c6457d5cc48","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3ea6fe4464b0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rwz8zs2wwdcn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3ea6fe4464b0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rwz8zs2wwdcn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"e1bcf6a02996f90595a0f9f4e4a9241d2b9ff98334f6b193cbdd8abe79433fa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2108331766317060096","occurred_at":"2026-10-08T23:00:38Z","failed_at":"e2e","signature_id":"a9cdc35fe4bd7f2b83c85c8090894982dc2b8c173bdc03255cf86269d22ffc5b","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2d03e6b69a40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-vv42cndqzc2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2d03e6b69a40\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-vv42cndqzc2q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108305902917390336","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T21:17:52Z","started_at":"2026-10-08T21:17:52Z","completed_at":"2026-10-08T23:30:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T23:48:24.543521521Z","timing_metadata_checked_at":"2026-10-08T23:48:24.543521521Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T23:48:24.547289309Z","region_metadata_checked_at":"2026-10-08T23:48:24.547289309Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108296013604196352","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T20:38:34Z","started_at":"2026-10-08T20:38:34Z","completed_at":"2026-10-08T21:09:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T21:11:10.865452063Z","timing_metadata_checked_at":"2026-10-08T21:11:10.865452063Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-08T21:11:10.868361155Z","region_metadata_checked_at":"2026-10-08T22:13:41.860924936Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ba528236be51c71c31fa625d3952e53e5a17edf1d0651b7f1c8ec6d76513d71b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108296013604196352","occurred_at":"2026-10-08T20:38:34Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T17:28:22Z","started_at":"2026-10-08T17:28:22Z","completed_at":"2026-10-08T20:12:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T20:28:40.945345149Z","timing_metadata_checked_at":"2026-10-08T20:28:40.945345149Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T20:28:40.949121438Z","region_metadata_checked_at":"2026-10-08T20:28:40.949121438Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"0d087ab08155503f2bc663360b9ff216aa5e408a20ab7a902033010f008503a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"e57178da6231f676c3543909d6927a0653c8f12c71f9a8924178794a486457a8","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e7b26aac4c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fbkb2qz6f9rf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9afe7411-35e5-4565-a7d7-5d4e665fe58f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9afe7411-35e5-4565-a7d7-5d4e665fe58f\",\n      \"name\": \"9afe7411-35e5-4565-a7d7-5d4e665fe58f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:03:48.271009954Z\",\n      \"endTime\": \"2026-10-08T18:32:55.449378647Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n to exist\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-phn9g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-phn9g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n to exist\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-phn9g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-phn9g to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e7b26aac4c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fbkb2qz6f9rf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9afe7411-35e5-4565-a7d7-5d4e665fe58f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/9afe7411-35e5-4565-a7d7-5d4e665fe58f\",\n      \"name\": \"9afe7411-35e5-4565-a7d7-5d4e665fe58f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:03:48.271009954Z\",\n      \"endTime\": \"2026-10-08T18:32:55.449378647Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n to exist\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-phn9g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-phn9g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-lfj4n to exist\\nMachine r9j7p3c0y1b3k9z-np-1-44qrg-phn9g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fbkb2qz6f9rf--managed/providers/Microsoft.Compute/virtualMachines/r9j7p3c0y1b3k9z-np-1-44qrg-phn9g to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"22572715b80e8017a8994f225991bfe917489f96bf2dfda9399609f30eb9bc63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"cbd310b6320cdff663edcfcdc2aaf12bce0f77d46d8accdace4e10db5e71773a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-cz595b\" for cluster \"cluster-candidate-4-22-cz595b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x18c3c8280a0\u003e: \n    failed to create NodePool np-cz595b: failed waiting for nodepool=\"np-cz595b\" for cluster \"cluster-candidate-4-22-cz595b\" in resourcegroup=\"rg-candidate-4-22-vzqcrpffcth2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7c22ea53-4deb-472f-837d-658a6f8cc25d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7c22ea53-4deb-472f-837d-658a6f8cc25d\",\n      \"name\": \"7c22ea53-4deb-472f-837d-658a6f8cc25d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:36:25.022726186Z\",\n      \"endTime\": \"2026-10-08T19:05:25.609844949Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9 to exist\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9 to exist\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-cz595b\" for cluster \"cluster-candidate-4-22-cz595b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x18c3c8280a0\u003e: \n    failed to create NodePool np-cz595b: failed waiting for nodepool=\"np-cz595b\" for cluster \"cluster-candidate-4-22-cz595b\" in resourcegroup=\"rg-candidate-4-22-vzqcrpffcth2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7c22ea53-4deb-472f-837d-658a6f8cc25d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/7c22ea53-4deb-472f-837d-658a6f8cc25d\",\n      \"name\": \"7c22ea53-4deb-472f-837d-658a6f8cc25d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:36:25.022726186Z\",\n      \"endTime\": \"2026-10-08T19:05:25.609844949Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9 to exist\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-drkw9 to exist\\nMachine f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-vzqcrpffcth2-candidate-cz595b--managed/providers/Microsoft.Compute/virtualMachines/f1m0r4q7i9z9q8o-np-cz595b-w6m8q-g48ht to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"61d521d0066674c9556d52aff325a12fbf9f5c2e2ebd8df1aa05ea70f1b6f1cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"9fc1a6a4fa883fc6875002b44e541ced39ee5185ee8b663d0df77575e08f0939","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fn9nq4\" for cluster \"cluster-candidate-4-21-fn9nq4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1aded3148440\u003e: \n    failed to create NodePool np-fn9nq4: failed waiting for nodepool=\"np-fn9nq4\" for cluster \"cluster-candidate-4-21-fn9nq4\" in resourcegroup=\"rg-candidate-4-21-ghsd6g8zdnw5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/70a9d05d-3834-496b-ab23-90c8635291e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/70a9d05d-3834-496b-ab23-90c8635291e9\",\n      \"name\": \"70a9d05d-3834-496b-ab23-90c8635291e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:35:08.455675057Z\",\n      \"endTime\": \"2026-10-08T19:04:15.635949535Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p to exist\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p to exist\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fn9nq4\" for cluster \"cluster-candidate-4-21-fn9nq4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1aded3148440\u003e: \n    failed to create NodePool np-fn9nq4: failed waiting for nodepool=\"np-fn9nq4\" for cluster \"cluster-candidate-4-21-fn9nq4\" in resourcegroup=\"rg-candidate-4-21-ghsd6g8zdnw5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/70a9d05d-3834-496b-ab23-90c8635291e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/70a9d05d-3834-496b-ab23-90c8635291e9\",\n      \"name\": \"70a9d05d-3834-496b-ab23-90c8635291e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:35:08.455675057Z\",\n      \"endTime\": \"2026-10-08T19:04:15.635949535Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p to exist\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-lsc2p to exist\\nMachine b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ghsd6g8zdnw5-candidate-fn9nq4--managed/providers/Microsoft.Compute/virtualMachines/b5c3z4o0q2e6o5q-np-fn9nq4-ttwck-vj8t5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6bcc7eb7e1f4e05a49d8aa4dd6e694939e625c1fbb505d3b5b17ce4e399b7bac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"30be14ccfca5a103a44375ac3df669c521021f4c93308b56a32de7bd14926922","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2f509ecfe3c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bw4mbhgcm827\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n      \"name\": \"d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:07:59.549473703Z\",\n      \"endTime\": \"2026-10-08T18:37:05.482678031Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2f509ecfe3c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bw4mbhgcm827\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n      \"name\": \"d4170a8b-86e1-4b07-aae0-9aa74a31f596\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:07:59.549473703Z\",\n      \"endTime\": \"2026-10-08T18:37:05.482678031Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cvbtc-4pk5m: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-4pk5m to exist\\nMachine cilium-cluster-cilium-np-cvbtc-mkchh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bw4mbhgcm827--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cvbtc-mkchh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"8645bfeb502dfcfc6d12093b2e514747378f78d09209b06c529eba6da0daa6a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"607737456265c716166bec4f750288f0594652400fdb685dc45a9ab0c9b032ba","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35ae7eca0680\u003e: \n    failed to create NodePool np-ntb88n: failed waiting for nodepool=\"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\" in resourcegroup=\"rg-candidate-5-1-tvlhq8pw8r2j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\",\n      \"name\": \"00bc71ab-da38-456c-8cc5-ad615c84b122\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:45:34.339197477Z\",\n      \"endTime\": \"2026-10-08T19:14:37.105782951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35ae7eca0680\u003e: \n    failed to create NodePool np-ntb88n: failed waiting for nodepool=\"np-ntb88n\" for cluster \"cluster-candidate-5-1-ntb88n\" in resourcegroup=\"rg-candidate-5-1-tvlhq8pw8r2j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/00bc71ab-da38-456c-8cc5-ad615c84b122\",\n      \"name\": \"00bc71ab-da38-456c-8cc5-ad615c84b122\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:45:34.339197477Z\",\n      \"endTime\": \"2026-10-08T19:14:37.105782951Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-48plr to exist\\nMachine z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tvlhq8pw8r2j-candidate-ntb88n--managed/providers/Microsoft.Compute/virtualMachines/z8r5l5c1h2t0o8m-np-ntb88n-lmxzn-hfnpk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b97b4c2d05d9dde28f9d1adfac53413e96d6c5f7db4de9e63a56c57a88dd4e41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"ecfced95d3fefee4d4fd692bafa6b115f8cb1c09453e7e1b0e30620a04bb8554","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3c50440e0018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nkr4c2vp869x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\",\n      \"name\": \"5de3f7d0-158f-4a7e-a885-89292635394e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:08:57.323157264Z\",\n      \"endTime\": \"2026-10-08T18:38:05.479695963Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3c50440e0018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nkr4c2vp869x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/5de3f7d0-158f-4a7e-a885-89292635394e\",\n      \"name\": \"5de3f7d0-158f-4a7e-a885-89292635394e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:08:57.323157264Z\",\n      \"endTime\": \"2026-10-08T18:38:05.479695963Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-ffrmf-nnf5q: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-ffrmf-shgcl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-nkr4c2vp869x--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-ffrmf-shgcl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"cb34c905d3813c4e6a2b253a589bddf1eaaac5b852a44074acf7a5fba8e73b42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"eae823e6b74a95a2b1e84a019161309a32d30986fbaa5b35e10457bb32383c4a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dlrjwb\" for cluster \"cluster-candidate-4-23-dlrjwb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b65d70d0400\u003e: \n    failed to create NodePool np-dlrjwb: failed waiting for nodepool=\"np-dlrjwb\" for cluster \"cluster-candidate-4-23-dlrjwb\" in resourcegroup=\"rg-candidate-4-23-sdtvdwbjbxfr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/79fc773d-2a37-4106-8f63-f194f928388d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/79fc773d-2a37-4106-8f63-f194f928388d\",\n      \"name\": \"79fc773d-2a37-4106-8f63-f194f928388d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:36:39.156679392Z\",\n      \"endTime\": \"2026-10-08T19:05:45.645587063Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z to exist\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z to exist\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dlrjwb\" for cluster \"cluster-candidate-4-23-dlrjwb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b65d70d0400\u003e: \n    failed to create NodePool np-dlrjwb: failed waiting for nodepool=\"np-dlrjwb\" for cluster \"cluster-candidate-4-23-dlrjwb\" in resourcegroup=\"rg-candidate-4-23-sdtvdwbjbxfr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/79fc773d-2a37-4106-8f63-f194f928388d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/79fc773d-2a37-4106-8f63-f194f928388d\",\n      \"name\": \"79fc773d-2a37-4106-8f63-f194f928388d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:36:39.156679392Z\",\n      \"endTime\": \"2026-10-08T19:05:45.645587063Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z to exist\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-96k5z to exist\\nMachine w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-sdtvdwbjbxfr-candidate-dlrjwb--managed/providers/Microsoft.Compute/virtualMachines/w2l2k7p1t2a1o2w-np-dlrjwb-79crd-vcm8b to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"da5efc0de8d584adc37d33c591b14f55637bbf91d135a302625dcfd3958a29e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"1de706bd8c752817d789e3f2ee0485ef15ce188abeabb1b4b45baa1c1e902769","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x553d3fef80\u003e: \n    failed to create NodePool np-kn7h4c: failed waiting for nodepool=\"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\" in resourcegroup=\"rg-candidate-5-0-5tp4pjp8hk9v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n      \"name\": \"b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:41:12.213982763Z\",\n      \"endTime\": \"2026-10-08T19:10:15.712133624Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x553d3fef80\u003e: \n    failed to create NodePool np-kn7h4c: failed waiting for nodepool=\"np-kn7h4c\" for cluster \"cluster-candidate-5-0-kn7h4c\" in resourcegroup=\"rg-candidate-5-0-5tp4pjp8hk9v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n      \"name\": \"b83f80b7-a3b4-4c91-9ea3-c3b3373ebcbb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:41:12.213982763Z\",\n      \"endTime\": \"2026-10-08T19:10:15.712133624Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-bs7p4 to exist\\nMachine b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-5tp4pjp8hk9v-candidate-kn7h4c--managed/providers/Microsoft.Compute/virtualMachines/b8y3z9m8y0d2g4x-np-kn7h4c-z99qt-pxbmm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"daea0d8bed6d012decb218d46bc677e81b282c11fb9a160400966d42a17c7225","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"8360fc4f2965ead66c616d75f5409d9c8aee0164b8d4d71a8ba9d6db0fc9f8aa","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3108b67364e0\u003e: \n    failed to create NodePool np-8mqkrm: failed waiting for nodepool=\"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\" in resourcegroup=\"rg-candidate-4-20-xmx872mb68nv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n      \"name\": \"827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:32:58.020047255Z\",\n      \"endTime\": \"2026-10-08T19:02:05.605513431Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3108b67364e0\u003e: \n    failed to create NodePool np-8mqkrm: failed waiting for nodepool=\"np-8mqkrm\" for cluster \"cluster-candidate-4-20-8mqkrm\" in resourcegroup=\"rg-candidate-4-20-xmx872mb68nv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n      \"name\": \"827a6af5-91f2-4a87-8008-85cec7a38eaf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:32:58.020047255Z\",\n      \"endTime\": \"2026-10-08T19:02:05.605513431Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-cv7mc to exist\\nMachine k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-xmx872mb68nv-candidate-8mqkrm--managed/providers/Microsoft.Compute/virtualMachines/k5p6u6p3d8e6w7b-np-8mqkrm-lk5dd-wvc7z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f47cc0aa84d55fc1eeeccb754a02990577c53a17bfbed3403966cca213c39e66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108248148466995200","occurred_at":"2026-10-08T17:28:22Z","failed_at":"e2e","signature_id":"8fae7edbf3fe4eba4c8d3ad2ab08abf901803a5c7eb69c8c4b81e4c6355c2ac1","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x237e16e43180\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-q28hc5z2kwsh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/6c8ad6b0-3620-4b89-8c2d-08b0866db056\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/6c8ad6b0-3620-4b89-8c2d-08b0866db056\",\n      \"name\": \"6c8ad6b0-3620-4b89-8c2d-08b0866db056\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:03:48.204928396Z\",\n      \"endTime\": \"2026-10-08T18:32:55.458196493Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-j8vmw-6lgjc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q28hc5z2kwsh--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-j8vmw-6lgjc to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-j8vmw-6lgjc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q28hc5z2kwsh--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-j8vmw-6lgjc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x237e16e43180\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-q28hc5z2kwsh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/6c8ad6b0-3620-4b89-8c2d-08b0866db056\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralindia/hcpOperationStatuses/6c8ad6b0-3620-4b89-8c2d-08b0866db056\",\n      \"name\": \"6c8ad6b0-3620-4b89-8c2d-08b0866db056\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T18:03:48.204928396Z\",\n      \"endTime\": \"2026-10-08T18:32:55.458196493Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-j8vmw-6lgjc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q28hc5z2kwsh--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-j8vmw-6lgjc to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-j8vmw-6lgjc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q28hc5z2kwsh--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-j8vmw-6lgjc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T13:52:10Z","started_at":"2026-10-08T13:52:10Z","completed_at":"2026-10-08T16:34:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T16:44:46.527369524Z","timing_metadata_checked_at":"2026-10-08T16:44:46.527369524Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T16:44:46.530695513Z","region_metadata_checked_at":"2026-10-08T16:44:46.530695513Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"608d32b47689dce6489539b93d853020c1792a101efd104a40762f168623ad07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"9b452c1be3c93f975397b5ccfaabffaec5e3ee43b442f86ff7704c7cfc6c3d23","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qzm2gk\" for cluster \"cluster-candidate-4-21-qzm2gk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39e342643720\u003e: \n    failed to create NodePool np-qzm2gk: failed waiting for nodepool=\"np-qzm2gk\" for cluster \"cluster-candidate-4-21-qzm2gk\" in resourcegroup=\"rg-candidate-4-21-t5wksgdrrckp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\",\n      \"name\": \"afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:53:33.939084658Z\",\n      \"endTime\": \"2026-10-08T15:22:35.324122041Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t to exist\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t to exist\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qzm2gk\" for cluster \"cluster-candidate-4-21-qzm2gk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39e342643720\u003e: \n    failed to create NodePool np-qzm2gk: failed waiting for nodepool=\"np-qzm2gk\" for cluster \"cluster-candidate-4-21-qzm2gk\" in resourcegroup=\"rg-candidate-4-21-t5wksgdrrckp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\",\n      \"name\": \"afe11fbe-72f7-4e4f-b45c-eb8ad99b3d2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:53:33.939084658Z\",\n      \"endTime\": \"2026-10-08T15:22:35.324122041Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t to exist\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-6tq9t to exist\\nMachine w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-t5wksgdrrckp-candidate-qzm2gk--managed/providers/Microsoft.Compute/virtualMachines/w9e3g8e6x8h0t9p-np-qzm2gk-lblkl-jlq5b to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6c516bd6cf84d1e62c1fd1a4c48407360afea9494a387eefb74627b46000210a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"77f4bb331d7c87c8c6f645d4b5f13f08407295d66ee31371666f5da75544d885","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x371874876980\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-ftnkcdsss4wg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/883d0d0d-c572-41f2-9f82-7d61a050db28\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/883d0d0d-c572-41f2-9f82-7d61a050db28\",\n      \"name\": \"883d0d0d-c572-41f2-9f82-7d61a050db28\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:27:29.613460957Z\",\n      \"endTime\": \"2026-10-08T14:56:34.939890509Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv to exist\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv to exist\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x371874876980\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-ftnkcdsss4wg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/883d0d0d-c572-41f2-9f82-7d61a050db28\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/883d0d0d-c572-41f2-9f82-7d61a050db28\",\n      \"name\": \"883d0d0d-c572-41f2-9f82-7d61a050db28\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:27:29.613460957Z\",\n      \"endTime\": \"2026-10-08T14:56:34.939890509Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv to exist\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-g7kbv to exist\\nMachine s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-ftnkcdsss4wg--managed/providers/Microsoft.Compute/virtualMachines/s6j5k7l8f8s0p3k-np-1-5bsqt-h5mb4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7f918c88fe9ad08671ad61d6e542244ee013a07babccaa8d5c4c7988234f7476","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"04197c3f7c272281d2f16a127440b51aecd5581858f715b36eb52e6c706ac7dc","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x6335433a090\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-tpwrtzjcnmjq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n      \"name\": \"3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:27:33.571990767Z\",\n      \"endTime\": \"2026-10-08T14:56:35.04864797Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-8swjn-292k4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-8swjn-292k4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x6335433a090\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-tpwrtzjcnmjq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n      \"name\": \"3feda0f9-93b1-4baa-b95d-bcd59b54cae3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:27:33.571990767Z\",\n      \"endTime\": \"2026-10-08T14:56:35.04864797Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-8swjn-292k4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-8swjn-b2vlb: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-8swjn-292k4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-tpwrtzjcnmjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-8swjn-292k4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"808fe86b6fa37a637289bcc47cc0556cffbf7f45c8d4d6e766be90d644a32592","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"cea1f06c65c9ac1a5240a3f2299004282ac5955daa892019aaf1e458ea109d8a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ctg8q4\" for cluster \"cluster-candidate-4-23-ctg8q4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ac3b36042c0\u003e: \n    failed to create NodePool np-ctg8q4: failed waiting for nodepool=\"np-ctg8q4\" for cluster \"cluster-candidate-4-23-ctg8q4\" in resourcegroup=\"rg-candidate-4-23-z87lsqfql56h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4467076c-8824-4b5f-8e60-3977a0fe20c4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4467076c-8824-4b5f-8e60-3977a0fe20c4\",\n      \"name\": \"4467076c-8824-4b5f-8e60-3977a0fe20c4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:57:55.181740971Z\",\n      \"endTime\": \"2026-10-08T15:27:00.08772636Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc to exist\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc to exist\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ctg8q4\" for cluster \"cluster-candidate-4-23-ctg8q4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3ac3b36042c0\u003e: \n    failed to create NodePool np-ctg8q4: failed waiting for nodepool=\"np-ctg8q4\" for cluster \"cluster-candidate-4-23-ctg8q4\" in resourcegroup=\"rg-candidate-4-23-z87lsqfql56h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4467076c-8824-4b5f-8e60-3977a0fe20c4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4467076c-8824-4b5f-8e60-3977a0fe20c4\",\n      \"name\": \"4467076c-8824-4b5f-8e60-3977a0fe20c4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:57:55.181740971Z\",\n      \"endTime\": \"2026-10-08T15:27:00.08772636Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc to exist\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-gdptc to exist\\nMachine l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-z87lsqfql56h-candidate-ctg8q4--managed/providers/Microsoft.Compute/virtualMachines/l7q9a4u1c8n5z2a-np-ctg8q4-cvxjg-kzdmt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8af9fc1debd8997228c5dc50c63c36c5235c97b4afe1a21148fad7a3d230225d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"6771b69c9cd8b6f46bf94cd51d146bf6cecc95e92b532d2cb269b24721b30ded","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3a6cffe3f060\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jct5pzjh8mxm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n      \"name\": \"b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T15:48:06.632742492Z\",\n      \"endTime\": \"2026-10-08T16:17:11.778582607Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3a6cffe3f060\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-jct5pzjh8mxm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n      \"name\": \"b64e41d5-253f-4190-baa0-1b895cdf4caf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T15:48:06.632742492Z\",\n      \"endTime\": \"2026-10-08T16:17:11.778582607Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-6hkbw to exist\\nMachine x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-jct5pzjh8mxm--managed/providers/Microsoft.Compute/virtualMachines/x0a2x4h1b1y4w4v-np-delete-main-8n5mr-wzs5x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management"}},{"row_id":"8b0c89e21557e2d717f89f8dbae5f26d924f555e9393c99fbd8b4ca42f7d7934","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"5f3bfb947330e190172ba5110d4435e2bf0388db1bc378c5cd2b5067c49ec984","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-24nxrk\" for cluster \"cluster-candidate-5-1-24nxrk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36b6bf7fec00\u003e: \n    failed to create NodePool np-24nxrk: failed waiting for nodepool=\"np-24nxrk\" for cluster \"cluster-candidate-5-1-24nxrk\" in resourcegroup=\"rg-candidate-5-1-74n6vxnsbzln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a208b4c2-0cec-4a7a-8f77-7140b3a3f092\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a208b4c2-0cec-4a7a-8f77-7140b3a3f092\",\n      \"name\": \"a208b4c2-0cec-4a7a-8f77-7140b3a3f092\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:58:11.889421655Z\",\n      \"endTime\": \"2026-10-08T15:27:20.151153667Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm to exist\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm to exist\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-24nxrk\" for cluster \"cluster-candidate-5-1-24nxrk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36b6bf7fec00\u003e: \n    failed to create NodePool np-24nxrk: failed waiting for nodepool=\"np-24nxrk\" for cluster \"cluster-candidate-5-1-24nxrk\" in resourcegroup=\"rg-candidate-5-1-74n6vxnsbzln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a208b4c2-0cec-4a7a-8f77-7140b3a3f092\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a208b4c2-0cec-4a7a-8f77-7140b3a3f092\",\n      \"name\": \"a208b4c2-0cec-4a7a-8f77-7140b3a3f092\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:58:11.889421655Z\",\n      \"endTime\": \"2026-10-08T15:27:20.151153667Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm to exist\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-4wfrm to exist\\nMachine p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-74n6vxnsbzln-candidate-24nxrk--managed/providers/Microsoft.Compute/virtualMachines/p6j0l4e0q2t9w1a-np-24nxrk-9749x-7hqz5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a9083cffcb27d3e238ee2b45350b0033e7e9c2b5ce19ab861e517d7ce6b8bc52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"78b9b25829be4c3984fe03924f28bc2a19d7a1e5fbcf3047833b3465f0d57419","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fgmnm8\" for cluster \"cluster-candidate-4-22-fgmnm8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ad692bfa2c0\u003e: \n    failed to create NodePool np-fgmnm8: failed waiting for nodepool=\"np-fgmnm8\" for cluster \"cluster-candidate-4-22-fgmnm8\" in resourcegroup=\"rg-candidate-4-22-t7czqbb9gdw6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df2fc222-a282-4c8a-885f-6afa0d259ffa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df2fc222-a282-4c8a-885f-6afa0d259ffa\",\n      \"name\": \"df2fc222-a282-4c8a-885f-6afa0d259ffa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T15:00:21.098278607Z\",\n      \"endTime\": \"2026-10-08T15:29:30.220139004Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp to exist\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp to exist\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fgmnm8\" for cluster \"cluster-candidate-4-22-fgmnm8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ad692bfa2c0\u003e: \n    failed to create NodePool np-fgmnm8: failed waiting for nodepool=\"np-fgmnm8\" for cluster \"cluster-candidate-4-22-fgmnm8\" in resourcegroup=\"rg-candidate-4-22-t7czqbb9gdw6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df2fc222-a282-4c8a-885f-6afa0d259ffa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df2fc222-a282-4c8a-885f-6afa0d259ffa\",\n      \"name\": \"df2fc222-a282-4c8a-885f-6afa0d259ffa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T15:00:21.098278607Z\",\n      \"endTime\": \"2026-10-08T15:29:30.220139004Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp to exist\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-fpkgp to exist\\nMachine o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-t7czqbb9gdw6-candidate-fgmnm8--managed/providers/Microsoft.Compute/virtualMachines/o5a0g0z5v8y7m7t-np-fgmnm8-mjz2d-l6qc5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b9c1d59bebd237518de05195e7ca383e1f832d136cdaf30d63742c9777710ec4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"d7c2cc02f3a607ef7b6cbe3b49264471171e7681498d80954c9331231b7a8a89","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x248495222160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-2s5lcfmgt854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/93a34e6e-477c-4e65-9e08-527e17028921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/93a34e6e-477c-4e65-9e08-527e17028921\",\n      \"name\": \"93a34e6e-477c-4e65-9e08-527e17028921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:23:16.915326226Z\",\n      \"endTime\": \"2026-10-08T14:52:17.249683042Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-xxpw5-lhcfb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2s5lcfmgt854--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-xxpw5-lhcfb to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-xxpw5-lhcfb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2s5lcfmgt854--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-xxpw5-lhcfb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x248495222160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-2s5lcfmgt854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/93a34e6e-477c-4e65-9e08-527e17028921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/93a34e6e-477c-4e65-9e08-527e17028921\",\n      \"name\": \"93a34e6e-477c-4e65-9e08-527e17028921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:23:16.915326226Z\",\n      \"endTime\": \"2026-10-08T14:52:17.249683042Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-xxpw5-lhcfb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2s5lcfmgt854--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-xxpw5-lhcfb to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-xxpw5-lhcfb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-2s5lcfmgt854--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-xxpw5-lhcfb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c15ae007646bc762cd892187b7b2b65201cecb0ceabb05c268d2087fb8625d51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"ccc88a84bcfc47cb169e8a095ef78886581bb9c45ae50908d9d1797d28040551","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fsgggm\" for cluster \"cluster-candidate-4-20-fsgggm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c37b4b9e200\u003e: \n    failed to create NodePool np-fsgggm: failed waiting for nodepool=\"np-fsgggm\" for cluster \"cluster-candidate-4-20-fsgggm\" in resourcegroup=\"rg-candidate-4-20-rz65zsts87v9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/240398ab-2ae4-4b7b-a355-04e9e2139068\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/240398ab-2ae4-4b7b-a355-04e9e2139068\",\n      \"name\": \"240398ab-2ae4-4b7b-a355-04e9e2139068\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:57:34.48901399Z\",\n      \"endTime\": \"2026-10-08T15:26:35.280704579Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn to exist\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn to exist\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fsgggm\" for cluster \"cluster-candidate-4-20-fsgggm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c37b4b9e200\u003e: \n    failed to create NodePool np-fsgggm: failed waiting for nodepool=\"np-fsgggm\" for cluster \"cluster-candidate-4-20-fsgggm\" in resourcegroup=\"rg-candidate-4-20-rz65zsts87v9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/240398ab-2ae4-4b7b-a355-04e9e2139068\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/240398ab-2ae4-4b7b-a355-04e9e2139068\",\n      \"name\": \"240398ab-2ae4-4b7b-a355-04e9e2139068\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:57:34.48901399Z\",\n      \"endTime\": \"2026-10-08T15:26:35.280704579Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn to exist\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-7xmfn to exist\\nMachine w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-rz65zsts87v9-candidate-fsgggm--managed/providers/Microsoft.Compute/virtualMachines/w4t7a3t7b9u9j2z-np-fsgggm-bz2bl-rjxzc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cf1eff9fbf03650fdced227e66686ce898bc70324b002192c474fe40982ab577","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"524a2a76e90c2d1da53a5c0a9582d861268831f07109dc1e7db02e6ed4141945","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2a3a9b00e6d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hxkght4ww89l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1026664-b5ca-42f5-adc7-85aaeef21d81\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1026664-b5ca-42f5-adc7-85aaeef21d81\",\n      \"name\": \"f1026664-b5ca-42f5-adc7-85aaeef21d81\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:28:38.920968051Z\",\n      \"endTime\": \"2026-10-08T14:57:44.886200601Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-8bbcc-kjgxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-kjgxq to exist\\nMachine cilium-cluster-cilium-np-8bbcc-krtkf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-krtkf to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-8bbcc-kjgxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-kjgxq to exist\\nMachine cilium-cluster-cilium-np-8bbcc-krtkf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-krtkf to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2a3a9b00e6d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hxkght4ww89l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1026664-b5ca-42f5-adc7-85aaeef21d81\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f1026664-b5ca-42f5-adc7-85aaeef21d81\",\n      \"name\": \"f1026664-b5ca-42f5-adc7-85aaeef21d81\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:28:38.920968051Z\",\n      \"endTime\": \"2026-10-08T14:57:44.886200601Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-8bbcc-kjgxq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-kjgxq to exist\\nMachine cilium-cluster-cilium-np-8bbcc-krtkf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-krtkf to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-8bbcc-kjgxq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-kjgxq to exist\\nMachine cilium-cluster-cilium-np-8bbcc-krtkf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hxkght4ww89l--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-8bbcc-krtkf to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e340178c0d8e0d0d1dc84cbc7a9fd1bd10bf04551585781678969d821f003b1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108193764467019776","occurred_at":"2026-10-08T13:52:10Z","failed_at":"e2e","signature_id":"ca182933bc653faf7eb30127aebd6ed595c5e97059da875779383f3b49535773","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-c7f6vs\" for cluster \"cluster-candidate-5-0-c7f6vs\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa91b8f312a0\u003e: \n    failed to create NodePool np-c7f6vs: failed waiting for nodepool=\"np-c7f6vs\" for cluster \"cluster-candidate-5-0-c7f6vs\" in resourcegroup=\"rg-candidate-5-0-mlpng9ckwtsq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/630087f9-3e91-4cc3-95cc-3a0875463d79\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/630087f9-3e91-4cc3-95cc-3a0875463d79\",\n      \"name\": \"630087f9-3e91-4cc3-95cc-3a0875463d79\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:56:46.681105188Z\",\n      \"endTime\": \"2026-10-08T15:25:55.347852762Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr to exist\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr to exist\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-c7f6vs\" for cluster \"cluster-candidate-5-0-c7f6vs\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa91b8f312a0\u003e: \n    failed to create NodePool np-c7f6vs: failed waiting for nodepool=\"np-c7f6vs\" for cluster \"cluster-candidate-5-0-c7f6vs\" in resourcegroup=\"rg-candidate-5-0-mlpng9ckwtsq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/630087f9-3e91-4cc3-95cc-3a0875463d79\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/630087f9-3e91-4cc3-95cc-3a0875463d79\",\n      \"name\": \"630087f9-3e91-4cc3-95cc-3a0875463d79\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T14:56:46.681105188Z\",\n      \"endTime\": \"2026-10-08T15:25:55.347852762Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr to exist\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-665jr to exist\\nMachine c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-mlpng9ckwtsq-candidate-c7f6vs--managed/providers/Microsoft.Compute/virtualMachines/c3n8k1n1b0g8r5z-np-c7f6vs-2zsgf-lnfgz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T03:46:27Z","started_at":"2026-10-08T03:46:27Z","completed_at":"2026-10-08T06:41:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T06:58:48.848958423Z","timing_metadata_checked_at":"2026-10-08T06:58:48.848958423Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T06:58:48.853270609Z","region_metadata_checked_at":"2026-10-08T06:58:48.853270609Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"0689ca5a0065e3d8823a728436aa2c2d412e543792f8a8e3f6fb0c02d473d41e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"c07cc1511d492c4af5906301b6ac8729ea5b2acd7f223c5094e456cf7e259b7e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-cmn25n\" for cluster \"cluster-candidate-5-0-cmn25n\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19e2b2505020\u003e: \n    failed to create NodePool np-cmn25n: failed waiting for nodepool=\"np-cmn25n\" for cluster \"cluster-candidate-5-0-cmn25n\" in resourcegroup=\"rg-candidate-5-0-lhc5fj69t9pm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6879bb7c-c254-4641-ba70-7b41bb04b080\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6879bb7c-c254-4641-ba70-7b41bb04b080\",\n      \"name\": \"6879bb7c-c254-4641-ba70-7b41bb04b080\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:52:10.585592043Z\",\n      \"endTime\": \"2026-10-08T05:21:19.889088931Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l to exist\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l to exist\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-cmn25n\" for cluster \"cluster-candidate-5-0-cmn25n\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19e2b2505020\u003e: \n    failed to create NodePool np-cmn25n: failed waiting for nodepool=\"np-cmn25n\" for cluster \"cluster-candidate-5-0-cmn25n\" in resourcegroup=\"rg-candidate-5-0-lhc5fj69t9pm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6879bb7c-c254-4641-ba70-7b41bb04b080\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6879bb7c-c254-4641-ba70-7b41bb04b080\",\n      \"name\": \"6879bb7c-c254-4641-ba70-7b41bb04b080\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:52:10.585592043Z\",\n      \"endTime\": \"2026-10-08T05:21:19.889088931Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l to exist\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-c2h6l to exist\\nMachine r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-lhc5fj69t9pm-candidate-cmn25n--managed/providers/Microsoft.Compute/virtualMachines/r6x9q5o3w7j6w0k-np-cmn25n-kb9xb-lcbf7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"321d9f3948c5613ff9f1f989e13aac53c1d34d5d42d91a0b942f14c62a289b86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"0e5d065bab30d941a1416295359f6c50ec9d353a91602de79cd9253e6bac9e86","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1f97b95fe138\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-zl88ncfz4jp4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\",\n      \"name\": \"4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:18:50.555017189Z\",\n      \"endTime\": \"2026-10-08T04:47:59.745762657Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-n67fb-47fdq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-47fdq to exist\\nMachine cilium-cluster-cilium-np-n67fb-jqb5d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-jqb5d to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-n67fb-47fdq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-47fdq to exist\\nMachine cilium-cluster-cilium-np-n67fb-jqb5d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-jqb5d to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1f97b95fe138\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-zl88ncfz4jp4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\",\n      \"name\": \"4a2491cc-5c5e-4c96-b7d8-ff0a13549b34\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:18:50.555017189Z\",\n      \"endTime\": \"2026-10-08T04:47:59.745762657Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-n67fb-47fdq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-47fdq to exist\\nMachine cilium-cluster-cilium-np-n67fb-jqb5d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-jqb5d to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-n67fb-47fdq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-47fdq to exist\\nMachine cilium-cluster-cilium-np-n67fb-jqb5d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-zl88ncfz4jp4--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-n67fb-jqb5d to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3656a44f641fb97a49d342f9557cfc341900c599d57160568ab694fe9bfd652d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"ca5e81422055ee4bd8d2f8df0fa691b5dc11cb7cece132e6083770e89a70662a","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x228c1da4a2e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-tff6vqrfwntq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/40cea923-3d28-46b1-b377-958e6c7d32be\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/40cea923-3d28-46b1-b377-958e6c7d32be\",\n      \"name\": \"40cea923-3d28-46b1-b377-958e6c7d32be\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:17:53.289622119Z\",\n      \"endTime\": \"2026-10-08T04:46:59.632942955Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-7vfqr-gvgx9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-tff6vqrfwntq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-7vfqr-gvgx9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-7vfqr-gvgx9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-tff6vqrfwntq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-7vfqr-gvgx9 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x228c1da4a2e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-tff6vqrfwntq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/40cea923-3d28-46b1-b377-958e6c7d32be\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/40cea923-3d28-46b1-b377-958e6c7d32be\",\n      \"name\": \"40cea923-3d28-46b1-b377-958e6c7d32be\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:17:53.289622119Z\",\n      \"endTime\": \"2026-10-08T04:46:59.632942955Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-7vfqr-gvgx9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-tff6vqrfwntq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-7vfqr-gvgx9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-7vfqr-gvgx9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-tff6vqrfwntq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-7vfqr-gvgx9 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"38a0864e490ba06214861a9d18cb9804384f31e1146a2bf35df81690d2dbd4ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"212d4cea3adecaa7fb0b5b48ed66e7c9510a248ab4a2550d0e77537380105fa1","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xbf2bf37c078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-rgbgvj2hmvgv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\",\n      \"name\": \"5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:21:09.226625204Z\",\n      \"endTime\": \"2026-10-08T04:50:09.941476385Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-gd5q9-ck6dg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-gd5q9-kqsvs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rgbgvj2hmvgv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-gd5q9-kqsvs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-gd5q9-ck6dg: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-gd5q9-kqsvs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rgbgvj2hmvgv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-gd5q9-kqsvs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xbf2bf37c078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-rgbgvj2hmvgv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\",\n      \"name\": \"5b9ce1ac-a1fe-4426-8a3c-1594d8368aaa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:21:09.226625204Z\",\n      \"endTime\": \"2026-10-08T04:50:09.941476385Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-gd5q9-ck6dg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-gd5q9-kqsvs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rgbgvj2hmvgv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-gd5q9-kqsvs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-gd5q9-ck6dg: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-gd5q9-kqsvs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-rgbgvj2hmvgv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-gd5q9-kqsvs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"489dfac5fc01d6e197f333a0cb9254116fdefe570bed484df3321871d4b07956","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"f6915c2acdb65b50b5f491323caf68c80329ffbf00cf123f661b2d3be53776ae","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2wf5g9\" for cluster \"cluster-candidate-5-1-2wf5g9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28a272d90ba0\u003e: \n    failed to create NodePool np-2wf5g9: failed waiting for nodepool=\"np-2wf5g9\" for cluster \"cluster-candidate-5-1-2wf5g9\" in resourcegroup=\"rg-candidate-5-1-tpzmdkt924rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\",\n      \"name\": \"acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:52:41.296917226Z\",\n      \"endTime\": \"2026-10-08T05:21:49.770952812Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl to exist\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl to exist\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2wf5g9\" for cluster \"cluster-candidate-5-1-2wf5g9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28a272d90ba0\u003e: \n    failed to create NodePool np-2wf5g9: failed waiting for nodepool=\"np-2wf5g9\" for cluster \"cluster-candidate-5-1-2wf5g9\" in resourcegroup=\"rg-candidate-5-1-tpzmdkt924rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\",\n      \"name\": \"acf03b6d-6e69-4e4c-b3e4-ac3e6038ad4b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:52:41.296917226Z\",\n      \"endTime\": \"2026-10-08T05:21:49.770952812Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl to exist\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-hr2rl to exist\\nMachine v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-tpzmdkt924rr-candidate-2wf5g9--managed/providers/Microsoft.Compute/virtualMachines/v2e2m0f1q9f1d4a-np-2wf5g9-lbj8r-vj5nq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"530cb9faba5bf5132137f74587df0222fb8c053cb05cf4db6c02b63255eefbac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"50a678b577dd6ec95a39c4930784d2f2f8df4c2e4deb6c817d27050680bc7bad","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5fp562\" for cluster \"cluster-candidate-4-22-5fp562\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1acc4afec200\u003e: \n    failed to create NodePool np-5fp562: failed waiting for nodepool=\"np-5fp562\" for cluster \"cluster-candidate-4-22-5fp562\" in resourcegroup=\"rg-candidate-4-22-x7bk4qfg72wt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df0ac87d-a82c-40b8-81b5-d723efa85d74\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df0ac87d-a82c-40b8-81b5-d723efa85d74\",\n      \"name\": \"df0ac87d-a82c-40b8-81b5-d723efa85d74\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:49:21.745083143Z\",\n      \"endTime\": \"2026-10-08T05:18:29.866533862Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt to exist\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt to exist\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5fp562\" for cluster \"cluster-candidate-4-22-5fp562\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1acc4afec200\u003e: \n    failed to create NodePool np-5fp562: failed waiting for nodepool=\"np-5fp562\" for cluster \"cluster-candidate-4-22-5fp562\" in resourcegroup=\"rg-candidate-4-22-x7bk4qfg72wt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df0ac87d-a82c-40b8-81b5-d723efa85d74\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/df0ac87d-a82c-40b8-81b5-d723efa85d74\",\n      \"name\": \"df0ac87d-a82c-40b8-81b5-d723efa85d74\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:49:21.745083143Z\",\n      \"endTime\": \"2026-10-08T05:18:29.866533862Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt to exist\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-qqbdt to exist\\nMachine v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-x7bk4qfg72wt-candidate-5fp562--managed/providers/Microsoft.Compute/virtualMachines/v1g0k7c0m8d2y7v-np-5fp562-585r2-t7452 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"868418b3541d83837afb2bf28b3129a904a615b0c35d6a0ff4f986bc1db7f86b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"e31120375fe20c3189a3f2c035d3f9c8bac3cf1283b6c17aad50ebf324b212ad","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x12d67242a00\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n      \"name\": \"1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T05:42:55.903350034Z\",\n      \"endTime\": \"2026-10-08T06:12:00.742498982Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x12d67242a00\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n      \"name\": \"1e7bf622-4bc7-4a43-8991-89c9b6f6d374\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T05:42:55.903350034Z\",\n      \"endTime\": \"2026-10-08T06:12:00.742498982Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-p97bl to exist\\nMachine n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-wdb7hgvdsm8w--managed/providers/Microsoft.Compute/virtualMachines/n3e8p9e9i8i5i1r-np-deletable-fbchf-rqjc7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management"}},{"row_id":"875996f784515e3801c3cd8bba5867d64e55f47b9f8805078f0000b6cceabba6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"8d541c9f52f89327eb524d55f2cbc5184388fedfbf5f16fa8cb7e24ae40f7f99","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cad2d10c0a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4xktww2hxb7t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7eed3c62-0b0c-4aed-9a04-226a72003a56\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7eed3c62-0b0c-4aed-9a04-226a72003a56\",\n      \"name\": \"7eed3c62-0b0c-4aed-9a04-226a72003a56\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:19:51.649780345Z\",\n      \"endTime\": \"2026-10-08T04:48:59.779189852Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7 to exist\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7 to exist\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3cad2d10c0a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4xktww2hxb7t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7eed3c62-0b0c-4aed-9a04-226a72003a56\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7eed3c62-0b0c-4aed-9a04-226a72003a56\",\n      \"name\": \"7eed3c62-0b0c-4aed-9a04-226a72003a56\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:19:51.649780345Z\",\n      \"endTime\": \"2026-10-08T04:48:59.779189852Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7 to exist\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-rz6v7 to exist\\nMachine b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4xktww2hxb7t--managed/providers/Microsoft.Compute/virtualMachines/b3j0e6f1d8b5y0c-np-1-fs9xj-t8fpj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9c7309cec05c5118a8a3cdc8dcc321027b2e593cf1616511ea3783d0cf55bf5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"bc2d06f2e3bbb11b62e2bdbb0371b19a15817c0acbbe41a8a602481250c62016","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nm4ft4\" for cluster \"cluster-candidate-4-20-nm4ft4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e5feac2cf20\u003e: \n    failed to create NodePool np-nm4ft4: failed waiting for nodepool=\"np-nm4ft4\" for cluster \"cluster-candidate-4-20-nm4ft4\" in resourcegroup=\"rg-candidate-4-20-gwp4bdzvdkfn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7d0b1c33-9150-4065-8244-327119876e65\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7d0b1c33-9150-4065-8244-327119876e65\",\n      \"name\": \"7d0b1c33-9150-4065-8244-327119876e65\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:47:03.009170999Z\",\n      \"endTime\": \"2026-10-08T05:16:09.831739591Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw to exist\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw to exist\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nm4ft4\" for cluster \"cluster-candidate-4-20-nm4ft4\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e5feac2cf20\u003e: \n    failed to create NodePool np-nm4ft4: failed waiting for nodepool=\"np-nm4ft4\" for cluster \"cluster-candidate-4-20-nm4ft4\" in resourcegroup=\"rg-candidate-4-20-gwp4bdzvdkfn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7d0b1c33-9150-4065-8244-327119876e65\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7d0b1c33-9150-4065-8244-327119876e65\",\n      \"name\": \"7d0b1c33-9150-4065-8244-327119876e65\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:47:03.009170999Z\",\n      \"endTime\": \"2026-10-08T05:16:09.831739591Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw to exist\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-c72fw to exist\\nMachine l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwp4bdzvdkfn-candidate-nm4ft4--managed/providers/Microsoft.Compute/virtualMachines/l0f1y9g7w2g8h1t-np-nm4ft4-rnsd9-vzhb4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b087af5f901e9c6fcea71002963c03e8728c722c9dce6b700d1fe4f3d21e8806","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"704461f11fba58d3264f2375448ec8d0dcfefed9be4b8d25b1805dc09d5f9d85","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kjsxrn\" for cluster \"cluster-candidate-4-23-kjsxrn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x27ed88bbf980\u003e: \n    failed to create NodePool np-kjsxrn: failed waiting for nodepool=\"np-kjsxrn\" for cluster \"cluster-candidate-4-23-kjsxrn\" in resourcegroup=\"rg-candidate-4-23-9blkbrrd87j2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ea16240d-6235-4ea8-8e63-3873031d6d51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ea16240d-6235-4ea8-8e63-3873031d6d51\",\n      \"name\": \"ea16240d-6235-4ea8-8e63-3873031d6d51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:54:34.518182875Z\",\n      \"endTime\": \"2026-10-08T05:23:35.635646755Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2 to exist\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2 to exist\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kjsxrn\" for cluster \"cluster-candidate-4-23-kjsxrn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x27ed88bbf980\u003e: \n    failed to create NodePool np-kjsxrn: failed waiting for nodepool=\"np-kjsxrn\" for cluster \"cluster-candidate-4-23-kjsxrn\" in resourcegroup=\"rg-candidate-4-23-9blkbrrd87j2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ea16240d-6235-4ea8-8e63-3873031d6d51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ea16240d-6235-4ea8-8e63-3873031d6d51\",\n      \"name\": \"ea16240d-6235-4ea8-8e63-3873031d6d51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:54:34.518182875Z\",\n      \"endTime\": \"2026-10-08T05:23:35.635646755Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2 to exist\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-6lpm2 to exist\\nMachine d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-9blkbrrd87j2-candidate-kjsxrn--managed/providers/Microsoft.Compute/virtualMachines/d9w5y9o4e7p2b1d-np-kjsxrn-79rwc-ct276 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c4d2d9d1b56975646ec96bf8c7f196d445714641cd5788d5adae0109a6f8033c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2108041306176491520","occurred_at":"2026-10-08T03:46:27Z","failed_at":"e2e","signature_id":"b9836bfff02572891050840308d6c8bb9be4b2b018e321a4135572ee218da36e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5bpm5c\" for cluster \"cluster-candidate-4-21-5bpm5c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39a1a2b36a20\u003e: \n    failed to create NodePool np-5bpm5c: failed waiting for nodepool=\"np-5bpm5c\" for cluster \"cluster-candidate-4-21-5bpm5c\" in resourcegroup=\"rg-candidate-4-21-vptzpz86kwlt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b709679-17d4-47e9-a453-c0f7086a9226\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b709679-17d4-47e9-a453-c0f7086a9226\",\n      \"name\": \"5b709679-17d4-47e9-a453-c0f7086a9226\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:50:41.026700325Z\",\n      \"endTime\": \"2026-10-08T05:19:49.834642675Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99 to exist\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99 to exist\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5bpm5c\" for cluster \"cluster-candidate-4-21-5bpm5c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39a1a2b36a20\u003e: \n    failed to create NodePool np-5bpm5c: failed waiting for nodepool=\"np-5bpm5c\" for cluster \"cluster-candidate-4-21-5bpm5c\" in resourcegroup=\"rg-candidate-4-21-vptzpz86kwlt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b709679-17d4-47e9-a453-c0f7086a9226\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b709679-17d4-47e9-a453-c0f7086a9226\",\n      \"name\": \"5b709679-17d4-47e9-a453-c0f7086a9226\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T04:50:41.026700325Z\",\n      \"endTime\": \"2026-10-08T05:19:49.834642675Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99 to exist\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-8lv99 to exist\\nMachine n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-vptzpz86kwlt-candidate-5bpm5c--managed/providers/Microsoft.Compute/virtualMachines/n5d8q2r3o3j4j9i-np-5bpm5c-n48fq-vf7nj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T02:00:37Z","started_at":"2026-10-08T02:00:37Z","completed_at":"2026-10-08T04:24:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T04:39:48.472775399Z","timing_metadata_checked_at":"2026-10-08T04:39:48.472775399Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T04:39:48.476543988Z","region_metadata_checked_at":"2026-10-08T04:39:48.476543988Z"},"failed_at":["e2e"],"failed_test_count":34,"occurrences":[{"row_id":"14966e10511ef08503c658c34d2b5114bc8c570733e90d6d7188aa635b328b7b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"11fb9973540ad640cafcdd9a78df56a498b2cd9d499c2536d6c8e235215b9784","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3bcb55dd85a0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-crphwg\" resourcegroup=\"rg-np-dg-minor-crphwg-6m9n6b7fcg8d\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-crphwg-6m9n6b7fcg8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-crphwg/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3bcb55dd85a0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-crphwg\" resourcegroup=\"rg-np-dg-minor-crphwg-6m9n6b7fcg8d\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-crphwg-6m9n6b7fcg8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-crphwg/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"186408d744410d84655ae04ddeda71286bbf193d5137fa4d86aec803863bc6bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"7bc1860acf1bdc160e1d9d3e083e6a77b4a25446ec861b06bce73c5aef74c5d0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8tzhmt\" for cluster \"cluster-candidate-4-22-8tzhmt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23e3b649efa0\u003e: \n    failed to create NodePool np-8tzhmt: failed waiting for nodepool=\"np-8tzhmt\" for cluster \"cluster-candidate-4-22-8tzhmt\" in resourcegroup=\"rg-candidate-4-22-sbbrjn549jd7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/33868d99-b424-4bf0-8b0a-debdff616e75\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/33868d99-b424-4bf0-8b0a-debdff616e75\",\n      \"name\": \"33868d99-b424-4bf0-8b0a-debdff616e75\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:47:01.121478067Z\",\n      \"endTime\": \"2026-10-08T03:16:05.005157282Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst to exist\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst to exist\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8tzhmt\" for cluster \"cluster-candidate-4-22-8tzhmt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x23e3b649efa0\u003e: \n    failed to create NodePool np-8tzhmt: failed waiting for nodepool=\"np-8tzhmt\" for cluster \"cluster-candidate-4-22-8tzhmt\" in resourcegroup=\"rg-candidate-4-22-sbbrjn549jd7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/33868d99-b424-4bf0-8b0a-debdff616e75\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/33868d99-b424-4bf0-8b0a-debdff616e75\",\n      \"name\": \"33868d99-b424-4bf0-8b0a-debdff616e75\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:47:01.121478067Z\",\n      \"endTime\": \"2026-10-08T03:16:05.005157282Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst to exist\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-gcgst to exist\\nMachine c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-sbbrjn549jd7-candidate-8tzhmt--managed/providers/Microsoft.Compute/virtualMachines/c1c7z9r9e9y0p3n-np-8tzhmt-skdcr-rwlxn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1bd558ac86b721d816be67679fb2a0e075e7fa69155d9a9d03fb6195f33ccb05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"dc21971b0a431404b75ca2aa3a8a2071c2727c9e220c888a3e332140a7c42a55","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wlzmmg\" for cluster \"cluster-candidate-5-1-wlzmmg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1052ae90d2e0\u003e: \n    failed to create NodePool np-wlzmmg: failed waiting for nodepool=\"np-wlzmmg\" for cluster \"cluster-candidate-5-1-wlzmmg\" in resourcegroup=\"rg-candidate-5-1-2rb9r54kz678\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/ba8e191c-888b-485b-8866-d49e0334cc7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/ba8e191c-888b-485b-8866-d49e0334cc7c\",\n      \"name\": \"ba8e191c-888b-485b-8866-d49e0334cc7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:51:25.134077752Z\",\n      \"endTime\": \"2026-10-08T03:20:27.169387826Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb to exist\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb to exist\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wlzmmg\" for cluster \"cluster-candidate-5-1-wlzmmg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1052ae90d2e0\u003e: \n    failed to create NodePool np-wlzmmg: failed waiting for nodepool=\"np-wlzmmg\" for cluster \"cluster-candidate-5-1-wlzmmg\" in resourcegroup=\"rg-candidate-5-1-2rb9r54kz678\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/ba8e191c-888b-485b-8866-d49e0334cc7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/ba8e191c-888b-485b-8866-d49e0334cc7c\",\n      \"name\": \"ba8e191c-888b-485b-8866-d49e0334cc7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:51:25.134077752Z\",\n      \"endTime\": \"2026-10-08T03:20:27.169387826Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb to exist\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-b2gkb to exist\\nMachine y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-2rb9r54kz678-candidate-wlzmmg--managed/providers/Microsoft.Compute/virtualMachines/y0d6e1a2j4h5z0e-np-wlzmmg-xvp24-kn8cg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2177f95f365731cbb0ea068d0849d2e35cbd9f556ca72e6613f3559b49b28e40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"6e99dc03e140d021122b5f99a0b39eb90363b2a3eb418fc13a8bcf957465177e","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x4fa9c508420\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-s9lxbp6g9wp9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x4fa9c508420\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-s9lxbp6g9wp9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"27a2e14421f895218bfd5bb145d8093811f0acb482eac4dd6c9c185e50d85a77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"9f77071dc6747b0314cd8920ca2287311b939a55c7de09fa54be0bfe2bcd1f78","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2523ecbc0ea0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zgfh95c7gz2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n      \"name\": \"cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T03:28:30.373260167Z\",\n      \"endTime\": \"2026-10-08T03:57:35.369426461Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2523ecbc0ea0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zgfh95c7gz2l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n      \"name\": \"cc6e942f-2673-4375-ba3b-1fa8f048b004\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T03:28:30.373260167Z\",\n      \"endTime\": \"2026-10-08T03:57:35.369426461Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-dp8nw to exist\\nMachine q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-zgfh95c7gz2l--managed/providers/Microsoft.Compute/virtualMachines/q0h7k4x2z6s6n0j-np-deletable-tp42z-gfntq to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdb7hgvdsm8w\" to finish creating: GET https://management"}},{"row_id":"2b77fd07245ac0d996ddbda292b0b13fde4308ef75ba39d22a347310328fd073","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"d9ecb82f70950229f902353084a6415477e147720c50fc7d251959e1cc792350","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-l5lgrv\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1c5666c0fce0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-hhxjcw4kwjpv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-l5lgrv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-l5lgrv\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1c5666c0fce0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-hhxjcw4kwjpv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-l5lgrv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2c3ec25b0d3f13a7fe9a1535fc096e04c42ca1919d375441490495bb85f80743","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"81166f4884fdcd117896960c8c39ef279bd4c30f7dfcfe06334faafd896c846c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-bd76nk\" on minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1ad51ba8c390\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-xg2dpqfdj5bt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-bd76nk\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-bd76nk\" on minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1ad51ba8c390\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-xg2dpqfdj5bt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-bd76nk\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"320e9ebe656e8a7cf72010184d0ace936716aa64d4a479519c6113d40fca1a21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"3979048c452a40f54f92711abf6015db7d4a3247bf1fce0134b9a35f9c83290f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-kdl52j\" on minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xed02447ca80\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-lljlhvtp7gqk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-kdl52j\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-kdl52j\" on minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xed02447ca80\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-lljlhvtp7gqk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-kdl52j\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"38afadc8f4353feef22f22bb58447a449c3516a7ac6e0105e16c5fd206646d31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"8dbd91ef71ec969c7b63355f72e7f62a344f89bc68b893cbfcc89d6c2e547cb5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tkcz74\" for cluster \"cluster-candidate-4-21-tkcz74\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb57001bd580\u003e: \n    failed to create NodePool np-tkcz74: failed waiting for nodepool=\"np-tkcz74\" for cluster \"cluster-candidate-4-21-tkcz74\" in resourcegroup=\"rg-candidate-4-21-qcxhk95mnhvk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/95120e83-d81e-45c6-bda2-21b985646077\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/95120e83-d81e-45c6-bda2-21b985646077\",\n      \"name\": \"95120e83-d81e-45c6-bda2-21b985646077\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:45:34.00484297Z\",\n      \"endTime\": \"2026-10-08T03:14:34.986063906Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx to exist\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx to exist\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tkcz74\" for cluster \"cluster-candidate-4-21-tkcz74\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb57001bd580\u003e: \n    failed to create NodePool np-tkcz74: failed waiting for nodepool=\"np-tkcz74\" for cluster \"cluster-candidate-4-21-tkcz74\" in resourcegroup=\"rg-candidate-4-21-qcxhk95mnhvk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/95120e83-d81e-45c6-bda2-21b985646077\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/95120e83-d81e-45c6-bda2-21b985646077\",\n      \"name\": \"95120e83-d81e-45c6-bda2-21b985646077\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:45:34.00484297Z\",\n      \"endTime\": \"2026-10-08T03:14:34.986063906Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx to exist\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-9fbdx to exist\\nMachine u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qcxhk95mnhvk-candidate-tkcz74--managed/providers/Microsoft.Compute/virtualMachines/u8v7l1b3e9b8x1w-np-tkcz74-jqkbr-l669g to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"39e49c472e534edf65f97e338eb433fbffcaa6834b0a46d75ea8454b7de89eb2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"3d17f8d6652d176f544f7ddbab6bf98de82badadb74a026139b33bc1b25c0814","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6b03fb724a0\u003e: \n    failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-4bzrjj\" resourcegroup=\"rg-np-noedge-4bzrjj-n97rz8gmwdvr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4bzrjj-n97rz8gmwdvr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-4bzrjj/nodePools/npnoedge-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x6b03fb724a0\u003e: \n    failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-4bzrjj\" resourcegroup=\"rg-np-noedge-4bzrjj-n97rz8gmwdvr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4bzrjj-n97rz8gmwdvr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-4bzrjj/nodePools/npnoedge-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"406c3dac759539db26f3e6fbecfb151a13c1309138a71e07be0da3341169f845","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"93c5f443b65f365a99e52dd59b3e31b6da3855ac65193c7fc2b456d85c0da930","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc813599cf60\u003e: \n    failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-dz427r\" resourcegroup=\"rg-np-skip-minor-dz427r-crp6tpsxwgq6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-dz427r-crp6tpsxwgq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-dz427r/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc813599cf60\u003e: \n    failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-dz427r\" resourcegroup=\"rg-np-skip-minor-dz427r-crp6tpsxwgq6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-dz427r-crp6tpsxwgq6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-dz427r/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"47502119e4a155db0b63d38c1ce3e9a1ce2bb015a9e8b8efdcbebc1ed3de3df2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"35790e776fa5175e4de78d7f6617ad98c96a8f158a3232735e601d4e5b778256","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-64zx6d\" to 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0xd58a144ed0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-dstvpqcxjmxk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-64zx6d\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-64zx6d\" to 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0xd58a144ed0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-dstvpqcxjmxk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-64zx6d\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4f1b777d68d2307c0345d55a91d231746a41a42a9d2fa63504fdca50d2154ef0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"399ce0931f87760834405b1f91f2bb0dc5c6dee722c1e4c00e390a98ef2e93ec","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2c220eff60c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-bxcw427xldqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f546f074-ba74-4d4b-ac5f-1b334996fa03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f546f074-ba74-4d4b-ac5f-1b334996fa03\",\n      \"name\": \"f546f074-ba74-4d4b-ac5f-1b334996fa03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:18:39.603761603Z\",\n      \"endTime\": \"2026-10-08T02:47:44.845130459Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-cxd7w-m4vt2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bxcw427xldqv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-cxd7w-m4vt2 to exist\\nMachine agg-cluster-agg-np-a-cxd7w-th9qk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-cxd7w-th9qk: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-cxd7w-m4vt2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bxcw427xldqv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-cxd7w-m4vt2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2c220eff60c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-bxcw427xldqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f546f074-ba74-4d4b-ac5f-1b334996fa03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f546f074-ba74-4d4b-ac5f-1b334996fa03\",\n      \"name\": \"f546f074-ba74-4d4b-ac5f-1b334996fa03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:18:39.603761603Z\",\n      \"endTime\": \"2026-10-08T02:47:44.845130459Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-cxd7w-m4vt2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bxcw427xldqv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-cxd7w-m4vt2 to exist\\nMachine agg-cluster-agg-np-a-cxd7w-th9qk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-cxd7w-th9qk: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-cxd7w-m4vt2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bxcw427xldqv--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-cxd7w-m4vt2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"540fe004db59d57632a6eb31c42efbb741c42918fba74f33543adce57ab98798","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"f6fc53eb549f0e7f2f95375c5420737172608e9674edf0be196e6c524d2b9c9a","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26e1bcba84a0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-7jfzmq\" resourcegroup=\"rg-np-downgrade-7jfzmq-rvr22sztdn98\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-7jfzmq-rvr22sztdn98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-7jfzmq/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26e1bcba84a0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-7jfzmq\" resourcegroup=\"rg-np-downgrade-7jfzmq-rvr22sztdn98\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-7jfzmq-rvr22sztdn98/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-7jfzmq/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"55307805d8a8b30cfdbf4a4b595402a521d427fa95b511f792a626f713c1eddd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"1609838bf5872d5e2d583f56af041bc1d204c93d24ccdb32dad101d6536f4ebf","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-hn2cnv\" on minor version 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0x9116922ea20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-4w8m65kzp5nn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-hn2cnv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-hn2cnv\" on minor version 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0x9116922ea20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-4w8m65kzp5nn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-hn2cnv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"699b0f65b5a1824d30c05839809075ac3099a2b3c1b8eeb56be0e7632810fc07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"f0bf9d5656cd4403fe7ec557e934ab25ebc30cdf02bf25bdc5454a196c5801bd","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-xjwrx8\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3f5c96b45080\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-lt88zk4qvv8g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-xjwrx8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-xjwrx8\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3f5c96b45080\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-lt88zk4qvv8g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-xjwrx8\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6b804eed762d5abeb2a719c1e2de9ff084e9104c02e3bb135ccd25901a849f42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"ddf416b60b13f8a7895320145ea311edeee5e97b0e336d70a0126bdfa4e1c5e4","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28f92e3a99c0\u003e: \n    failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-tw9ncc\" resourcegroup=\"rg-np-version-upgrade-tw9ncc-98zjd59jkjpr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tw9ncc-98zjd59jkjpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tw9ncc/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x28f92e3a99c0\u003e: \n    failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-tw9ncc\" resourcegroup=\"rg-np-version-upgrade-tw9ncc-98zjd59jkjpr\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tw9ncc-98zjd59jkjpr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tw9ncc/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6d1cb8c8f743985adb1b2f2947b9f876135b55a694f240adc8d8bb997127ef4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"d9b09315f5cbffbd9b174578ef412adca169d6ebcfe103e2e14792ace908df4b","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x65e48efd800\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-qwp7bbpclw7g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e1feb34-d8de-476b-993b-80f75f40bd9e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e1feb34-d8de-476b-993b-80f75f40bd9e\",\n      \"name\": \"1e1feb34-d8de-476b-993b-80f75f40bd9e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:19:11.90853804Z\",\n      \"endTime\": \"2026-10-08T02:48:14.711261391Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-p24md: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-p24md to exist\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-p24md: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-p24md to exist\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x65e48efd800\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-qwp7bbpclw7g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e1feb34-d8de-476b-993b-80f75f40bd9e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1e1feb34-d8de-476b-993b-80f75f40bd9e\",\n      \"name\": \"1e1feb34-d8de-476b-993b-80f75f40bd9e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:19:11.90853804Z\",\n      \"endTime\": \"2026-10-08T02:48:14.711261391Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-p24md: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-p24md to exist\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-p24md: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-p24md to exist\\nMachine e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-qwp7bbpclw7g--managed/providers/Microsoft.Compute/virtualMachines/e1v5o5b0z6h4y7i-np-1-xh5kh-q4hwl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"786928115e7b85b965612e45ed5d08b9e7c17fb5987ee1cf25a3528656e6456f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"3432fe66c4ed90923c24f3ae7f15c3a4bfc33f20470b8a1b2b57487830b17e6b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-x2nmgc\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3577745b47e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-l49w7x6sf8lk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-x2nmgc\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-x2nmgc\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3577745b47e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-l49w7x6sf8lk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-x2nmgc\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8a29209d4da02b2a146bed074ed7c21a78ba34f0531b211eee146507a6ec9548","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"697bacd58389e5a2c004217397ccee804ff03a55b0df4328e472ad3b6538aa9d","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35b2e2ff91e0\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-rx2rkxms8sw7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-rx2rkxms8sw7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35b2e2ff91e0\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-rx2rkxms8sw7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-rx2rkxms8sw7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"902867b8d922a670ddb97e92b74160c752642686b8f29b0615242c7618b7b694","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"4ea5e665e1441c36ffd5c3ff6c454b7210f8c32ad40846122aa42883cbc59638","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-wps54w\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3ddfbef4d200\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-9qcq2bc8pfhn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-wps54w\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-wps54w\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3ddfbef4d200\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-9qcq2bc8pfhn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-wps54w\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"91294f3642fd3e7467f1226eb5fec14c8a86fb66ed7d8476319a11ffb18aad77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"286f2b337db1c06afe4f552fafe360d2df35c905db51dc5712e0039ace5c0cf3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-znb5qj\" for cluster \"cluster-candidate-5-0-znb5qj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17bbc4694220\u003e: \n    failed to create NodePool np-znb5qj: failed waiting for nodepool=\"np-znb5qj\" for cluster \"cluster-candidate-5-0-znb5qj\" in resourcegroup=\"rg-candidate-5-0-d5trjc74qhsj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/14de4838-40ca-415c-a29b-7d78f06ecdac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/14de4838-40ca-415c-a29b-7d78f06ecdac\",\n      \"name\": \"14de4838-40ca-415c-a29b-7d78f06ecdac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:49:59.670336775Z\",\n      \"endTime\": \"2026-10-08T03:19:07.230773535Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82 to exist\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82 to exist\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-znb5qj\" for cluster \"cluster-candidate-5-0-znb5qj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17bbc4694220\u003e: \n    failed to create NodePool np-znb5qj: failed waiting for nodepool=\"np-znb5qj\" for cluster \"cluster-candidate-5-0-znb5qj\" in resourcegroup=\"rg-candidate-5-0-d5trjc74qhsj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/14de4838-40ca-415c-a29b-7d78f06ecdac\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/14de4838-40ca-415c-a29b-7d78f06ecdac\",\n      \"name\": \"14de4838-40ca-415c-a29b-7d78f06ecdac\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:49:59.670336775Z\",\n      \"endTime\": \"2026-10-08T03:19:07.230773535Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82 to exist\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-gpw82 to exist\\nMachine l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-d5trjc74qhsj-candidate-znb5qj--managed/providers/Microsoft.Compute/virtualMachines/l1n2h0z1b3o2j0p-np-znb5qj-cx29j-sxz4v to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"91a8fb037681201f0071057c78e2f37fc7a1549554fe857363d49c520cbc690c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"71ce598d6d9d958fde2cd619eb779c67cc6e323a3cce0328ebb3cc5eff50003c","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a6c3876d160\u003e: \n    failed waiting for deployment \"mi-missing-mi-hcp-cluster-8qb82d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-8qb82d/operationStatuses/08584101764787793835\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:101]: failed to deploy dedicated managed identities\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a6c3876d160\u003e: \n    failed waiting for deployment \"mi-missing-mi-hcp-cluster-8qb82d\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-missing-mi-hcp-cluster-8qb82d/operationStatuses/08584101764787793835\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterNonMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.Resources/deployments/clusterMsiScopedAssignments-jaooh6lj24n4o\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"NotFound\\\",\\r\\n            \\\"message\\\": \\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/delete-mi-rg-zm56p642vwdr/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8da18459eef6017d12822e9dd3b87c09fdb695ab8d9cb4881da243c125e2db6b","failure_pattern":"ERROR CODE: DeploymentFailed; detail code NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service-missing-mi-hcp-cluster' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"926d351375051e89ff778a8ba9a0efe4eac5609a965e70c5e6db97f7c3254f9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"49bc5c5cdc05073630918f616f1b204bce3a50fc2a76296971ab18ebc0822591","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x6f28c1d0c00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-vpk5q2z5574v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x6f28c1d0c00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-vpk5q2z5574v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"adf4d43683d33e78991e33268ad452577c023245ad376952ed4efbb985f06ff8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"e64509f96f2f07ea596d1a6b68e536a246bfe0c47479f375bb73c60def84644c","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x137051db4300\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-hcx5lwfp6892/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x137051db4300\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-hcx5lwfp6892/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b2ad1169a1cb2a22fcf63f820b28f514d60dec0e7b7884b5c76269a16b9c8f7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"97e9cfb4d9ae6fd232412678c19b7ea57ae3f6d1c4164518dd3f623a9673d2b5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xhn5bs\" for cluster \"cluster-candidate-4-23-xhn5bs\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31d2ef35d8e0\u003e: \n    failed to create NodePool np-xhn5bs: failed waiting for nodepool=\"np-xhn5bs\" for cluster \"cluster-candidate-4-23-xhn5bs\" in resourcegroup=\"rg-candidate-4-23-4llsqxjhkgt4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/93da1929-fde9-4a74-85a6-ad361a679f40\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/93da1929-fde9-4a74-85a6-ad361a679f40\",\n      \"name\": \"93da1929-fde9-4a74-85a6-ad361a679f40\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:49:02.43694261Z\",\n      \"endTime\": \"2026-10-08T03:18:07.302711356Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr to exist\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr to exist\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xhn5bs\" for cluster \"cluster-candidate-4-23-xhn5bs\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31d2ef35d8e0\u003e: \n    failed to create NodePool np-xhn5bs: failed waiting for nodepool=\"np-xhn5bs\" for cluster \"cluster-candidate-4-23-xhn5bs\" in resourcegroup=\"rg-candidate-4-23-4llsqxjhkgt4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/93da1929-fde9-4a74-85a6-ad361a679f40\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/93da1929-fde9-4a74-85a6-ad361a679f40\",\n      \"name\": \"93da1929-fde9-4a74-85a6-ad361a679f40\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:49:02.43694261Z\",\n      \"endTime\": \"2026-10-08T03:18:07.302711356Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr to exist\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-bjrwr to exist\\nMachine t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4llsqxjhkgt4-candidate-xhn5bs--managed/providers/Microsoft.Compute/virtualMachines/t7o7j7t1z4g4t0m-np-xhn5bs-lm5cm-gcr5k to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b7fdc8faa0cae5129368e672d232ca363da3b30023302654984e53151b773744","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"a502c3c33e95830c6b055e8b63c99d465d434e71cd54d550ac664ae0376a0a52","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2edd5c272b00\u003e: \n    failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-df62qq\" resourcegroup=\"rg-np-version-upgrade-df62qq-l9cc8d44b2p9\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-df62qq-l9cc8d44b2p9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-df62qq/nodePools/npupgrade-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2edd5c272b00\u003e: \n    failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-df62qq\" resourcegroup=\"rg-np-version-upgrade-df62qq-l9cc8d44b2p9\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-df62qq-l9cc8d44b2p9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-df62qq/nodePools/npupgrade-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b8b1332ab3598b4e81c31f5b81723ee5d77d00be4dfcf92b9e2f8f5408302135","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"8eb81065221254a41909e446e130e2c5564d04ecbae268b12a83dd5adbb708e4","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b63d32ea960\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-6l2qnmrqmq6s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/987e3697-e775-428f-b96b-007cf98cd200\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/987e3697-e775-428f-b96b-007cf98cd200\",\n      \"name\": \"987e3697-e775-428f-b96b-007cf98cd200\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:17:06.289611141Z\",\n      \"endTime\": \"2026-10-08T02:46:14.822515799Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-b4b48-6xc2t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-6l2qnmrqmq6s--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-b4b48-6xc2t to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-b4b48-6xc2t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-6l2qnmrqmq6s--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-b4b48-6xc2t to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b63d32ea960\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-6l2qnmrqmq6s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/987e3697-e775-428f-b96b-007cf98cd200\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/987e3697-e775-428f-b96b-007cf98cd200\",\n      \"name\": \"987e3697-e775-428f-b96b-007cf98cd200\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:17:06.289611141Z\",\n      \"endTime\": \"2026-10-08T02:46:14.822515799Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-b4b48-6xc2t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-6l2qnmrqmq6s--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-b4b48-6xc2t to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-b4b48-6xc2t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-6l2qnmrqmq6s--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-b4b48-6xc2t to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c141a316ba7b115d1fd98d2a343639c8170a433641ad2914b822a5d85e840df5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"66b6c637f092b0a1ae34328f0c47a9d939259c22ca517a9d0e2276a87a3b13b0","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-xf8crr\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0x28f031e29260\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-mqfd4qzn4578/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-xf8crr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-xf8crr\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0x28f031e29260\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-mqfd4qzn4578/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-xf8crr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c272f364b73991a037f7d63fa1c19bfbcd79db8802deff82f0ac36a869071f33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"363bec732de5d54b45f82f6c6410eef0cdb5ff0af00c3fc2788bcb78b1e622e4","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xe292679e030\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fwdf5gjhw97g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e74aa6d0-c8cd-430c-8190-549622080499\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e74aa6d0-c8cd-430c-8190-549622080499\",\n      \"name\": \"e74aa6d0-c8cd-430c-8190-549622080499\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:18:33.237818266Z\",\n      \"endTime\": \"2026-10-08T02:47:34.745399444Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-dc67w-8c8mq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-8c8mq to exist\\nMachine cilium-cluster-cilium-np-dc67w-p8qqc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-p8qqc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-dc67w-8c8mq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-8c8mq to exist\\nMachine cilium-cluster-cilium-np-dc67w-p8qqc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-p8qqc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xe292679e030\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fwdf5gjhw97g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e74aa6d0-c8cd-430c-8190-549622080499\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e74aa6d0-c8cd-430c-8190-549622080499\",\n      \"name\": \"e74aa6d0-c8cd-430c-8190-549622080499\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:18:33.237818266Z\",\n      \"endTime\": \"2026-10-08T02:47:34.745399444Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-dc67w-8c8mq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-8c8mq to exist\\nMachine cilium-cluster-cilium-np-dc67w-p8qqc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-p8qqc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-dc67w-8c8mq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-8c8mq to exist\\nMachine cilium-cluster-cilium-np-dc67w-p8qqc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-fwdf5gjhw97g--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-dc67w-p8qqc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c286511b3464afb3bafc9a8e0ba470fb745b7cdd6e8c608c87d498f3d7dccfb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"e088dc0de9a1fac4cac481a667a95c6c5ae11cc93acca3216cdc27fe06be74e3","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-pcrbcd\" on minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0x33c001cb49c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-zb4tmg42vgq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-pcrbcd\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-pcrbcd\" on minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0x33c001cb49c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-zb4tmg42vgq7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-pcrbcd\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c95790ba8f7de78ac9b61dbb2e0d12efd08af034e14ee49a49d3889434fdb858","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"b8dc64cff9c94bedff3d6968dd7a64f15705facf6a41874e487dabb240ea1551","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19c5b50bc860\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-x2784j5xhvrh\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-x2784j5xhvrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0x19c5b50bc860\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-x2784j5xhvrh\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-x2784j5xhvrh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"cda5197cc3247bba4a9f43335d43a6a3400bef72be79e52d7582b5dcc6b97dc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"a2308670e8d0cf661e1d2de3f880177522e71068292186eb5069f5861f9c634c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-gm6fx9\" for cluster \"cluster-candidate-4-20-gm6fx9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x130c0be607c0\u003e: \n    failed to create NodePool np-gm6fx9: failed waiting for nodepool=\"np-gm6fx9\" for cluster \"cluster-candidate-4-20-gm6fx9\" in resourcegroup=\"rg-candidate-4-20-ts84fqwf8d5v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9435cae-155f-47ef-8d90-e886b71aff15\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9435cae-155f-47ef-8d90-e886b71aff15\",\n      \"name\": \"b9435cae-155f-47ef-8d90-e886b71aff15\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:45:36.63094836Z\",\n      \"endTime\": \"2026-10-08T03:14:44.806443674Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj to exist\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj to exist\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-gm6fx9\" for cluster \"cluster-candidate-4-20-gm6fx9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x130c0be607c0\u003e: \n    failed to create NodePool np-gm6fx9: failed waiting for nodepool=\"np-gm6fx9\" for cluster \"cluster-candidate-4-20-gm6fx9\" in resourcegroup=\"rg-candidate-4-20-ts84fqwf8d5v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9435cae-155f-47ef-8d90-e886b71aff15\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/b9435cae-155f-47ef-8d90-e886b71aff15\",\n      \"name\": \"b9435cae-155f-47ef-8d90-e886b71aff15\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T02:45:36.63094836Z\",\n      \"endTime\": \"2026-10-08T03:14:44.806443674Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj to exist\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7c4lj to exist\\nMachine p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ts84fqwf8d5v-candidate-gm6fx9--managed/providers/Microsoft.Compute/virtualMachines/p6i3y8w1a7s1v4y-np-gm6fx9-xdhhm-7vpdv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f2deb49c8a8aca27af34ff3c7804bda6c0614ebda96975b4e4318262681ce1ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2108014673302392832","occurred_at":"2026-10-08T02:00:37Z","failed_at":"e2e","signature_id":"759bd9f85e38e37d1860e839ceb34b5c137190cb65b706eb8146bc3e9eb17622","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa63e00795e0\u003e: \n    failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-dwrcvf\" resourcegroup=\"rg-np-version-upgrade-dwrcvf-tndq67k8xt89\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dwrcvf-tndq67k8xt89/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-dwrcvf/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa63e00795e0\u003e: \n    failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-dwrcvf\" resourcegroup=\"rg-np-version-upgrade-dwrcvf-tndq67k8xt89\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dwrcvf-tndq67k8xt89/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-dwrcvf/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":34,"failure_pattern_ids":["07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","8da18459eef6017d12822e9dd3b87c09fdb695ab8d9cb4881da243c125e2db6b","a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610"],"matched_occurrences":34,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}