{"meta":{"date":"2026-10-21","timezone":"UTC","generated_at":"2026-10-29T00:50:19Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":12,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":9,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113042690441482240","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-21T23:00:10Z","started_at":"2026-10-21T23:00:10Z","completed_at":"2026-10-22T00:52:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T01:02:56.205305561Z","timing_metadata_checked_at":"2026-10-22T01:02:56.205305561Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T01:02:56.213597501Z","region_metadata_checked_at":"2026-10-22T01:02:56.213597501Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"97a1288b036b1e810aac7d12a859f940eb55176b6f59acfa0bd9e91530898e96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113042690441482240","occurred_at":"2026-10-21T23:00:10Z","failed_at":"e2e","signature_id":"c4eb20e93ae75373c59464f00b68f59434fe607f1d433e291947a1182f458028","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:349]: Timed out after 307.279s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:334 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0x337cde092010\u003e: \n    E1021 23:30:50.528002    2868 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    {\n        s: \"E1021 23:30:50.528002    2868 memcache.go:381] \\\"Couldn't get current server API group list\\\" err=\\\"the server has asked for the client to provide credentials\\\"\",\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:349]: Timed out after 307.279s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:334 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0x337cde092010\u003e: \n    E1021 23:30:50.528002    2868 memcache.go:381] \"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"\n    {\n        s: \"E1021 23:30:50.528002    2868 memcache.go:381] \\\"Couldn't get current server API group list\\\" err=\\\"the server has asked for the client to provide credentials\\\"\",\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"908166ee7c49cf07f1a2b5b18714d6e5f70255daa4c1d99ba634878b4c3b8017","failure_pattern":"\"Couldn't get current server API group list\" err=\"the server has asked for the client to provide credentials\"","search_query":"occurred"}},{"row_id":"ebe3135be18dd14bfac1bdede3469d095a1c78c0c612e7d1adb349baf3372285","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2113042690441482240","occurred_at":"2026-10-21T23:00:10Z","failed_at":"e2e","signature_id":"c7f43354740e69178951a6c68e22539a605df8b0aa7742a05b7005bd1faa97a1","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb243792ee20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xb243792edc0\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:297]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb243792ee20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0xb243792edc0\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: context deadline exceeded\",\n            err: \u003ccontext.deadlineExceededError\u003e{},\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b6694bcdf5bd78471f8a6f6a55664c9b2d5e3ffa6e0b99c19a0b7d948e44b177","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded","search_query":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.002s: failed to list pods in namespace pullsecret-image-test: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["908166ee7c49cf07f1a2b5b18714d6e5f70255daa4c1d99ba634878b4c3b8017","b6694bcdf5bd78471f8a6f6a55664c9b2d5e3ffa6e0b99c19a0b7d948e44b177"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113033513308524544","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-21T22:23:42Z","started_at":"2026-10-21T22:23:42Z","completed_at":"2026-10-22T00:46:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T00:52:03.15272358Z","timing_metadata_checked_at":"2026-10-22T00:52:03.15272358Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T00:52:03.155756272Z","region_metadata_checked_at":"2026-10-22T00:52:03.155756272Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"934119ef4db1f9d67ff3341a690f60fe714d0490f467c7b33e0f39b9a9ae6587","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113033513308524544","occurred_at":"2026-10-21T22:23:42Z","failed_at":"e2e","signature_id":"3aee454bda0345aa80151b07bb5e4c51af83dae06c70040ad111bd3819e733e7","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3866dcc37c20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\n      - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/ubi9/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        err: \u003c*errors.errorString | 0x3866dc67cd90\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3866dcc37c20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\n      - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \"registry.redhat.io/ubi9/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        err: \u003c*errors.errorString | 0x3866dc67cd90\u003e{\n            s: \"expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:\\n  - pod pull-secret-test container pull-test: ImagePullBackOff - Back-off pulling image \\\"registry.redhat.io/ubi9/ubi-minimal:latest\\\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://registry.redhat.io/ubi9/ubi-minimal:latest: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable; artifact err: get manifest: build image source: Requesting bearer token: received unexpected HTTP status: 503 Service Unavailable\",\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6de12f7229509a8873177b6aee486cac0c911989cb81031037d884ebc7dad3dd","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:","search_query":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.005s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: image pull errors:"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6de12f7229509a8873177b6aee486cac0c911989cb81031037d884ebc7dad3dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113031572314656768","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-21T22:15:59Z","started_at":"2026-10-21T22:15:59Z","completed_at":"2026-10-22T00:41:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T00:52:02.933671056Z","timing_metadata_checked_at":"2026-10-22T00:52:02.933671056Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T00:52:02.936556249Z","region_metadata_checked_at":"2026-10-22T00:52:02.936556249Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113031190301642752","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-21T22:14:28Z","started_at":"2026-10-21T22:14:28Z","completed_at":"2026-10-22T00:41:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T00:52:02.802617601Z","timing_metadata_checked_at":"2026-10-22T00:52:02.802617601Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T00:52:02.807602388Z","region_metadata_checked_at":"2026-10-22T00:52:02.807602388Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2113029944064872448","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-21T22:09:30Z","started_at":"2026-10-21T22:09:30Z","completed_at":"2026-10-22T00:34:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-22T00:41:24.935422043Z","timing_metadata_checked_at":"2026-10-22T00:41:24.935422043Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-22T00:41:24.942008625Z","region_metadata_checked_at":"2026-10-22T00:41:24.942008625Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112984313044668416","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-21T19:08:12Z","started_at":"2026-10-21T19:08:12Z","completed_at":"2026-10-21T21:21:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-21T21:32:53.677119083Z","timing_metadata_checked_at":"2026-10-21T21:32:53.677119083Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-21T21:32:53.680523672Z","region_metadata_checked_at":"2026-10-21T21:32:53.680523672Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112937268917309440","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-21T16:01:15Z","started_at":"2026-10-21T16:01:15Z","completed_at":"2026-10-21T18:26:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-21T18:33:28.930516787Z","timing_metadata_checked_at":"2026-10-21T18:33:28.930516787Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-21T18:33:28.933433775Z","region_metadata_checked_at":"2026-10-21T18:33:28.933433775Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bd362eebf7fec82df4d8b2d8b5850ade61f2f54a956ba82a4f05057a0a3555d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112937268917309440","occurred_at":"2026-10-21T16:01:15Z","failed_at":"e2e","signature_id":"6f8fa1557de794cf4f88a42a92286c03dca64dacbbf713b8b027b7822d5e2b70","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x29388dc21b20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-l6ddnj7vhmkq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\",\n      \"name\": \"8669d017-4213-4ec1-86fe-07c71e8067ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T16:34:55.668662338Z\",\n      \"endTime\": \"2026-10-21T16:53:59.759177283Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-l6ddnj7vhmkq--managed/providers/Microsoft.Compute/virtualMachines/s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-l6ddnj7vhmkq--managed/providers/Microsoft.Compute/virtualMachines/s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\n            correlation ID: 7627942b-6e2a-4d1f-a2c9-7a7ff2104478\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x29388dc21b20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-l6ddnj7vhmkq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\",\n      \"name\": \"8669d017-4213-4ec1-86fe-07c71e8067ec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-21T16:34:55.668662338Z\",\n      \"endTime\": \"2026-10-21T16:53:59.759177283Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-l6ddnj7vhmkq--managed/providers/Microsoft.Compute/virtualMachines/s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-l6ddnj7vhmkq--managed/providers/Microsoft.Compute/virtualMachines/s3e2z4z8o4i6e7y-np-1-7pccj-mpwmx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8669d017-4213-4ec1-86fe-07c71e8067ec\n            correlation ID: 7627942b-6e2a-4d1f-a2c9-7a7ff2104478\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f60b238f5515effd82f43ffe3aad7999950bc7d85bdfd2cdae8fca166b68f9de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f60b238f5515effd82f43ffe3aad7999950bc7d85bdfd2cdae8fca166b68f9de"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112778395547865088","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-21T05:29:57Z","started_at":"2026-10-21T05:29:57Z","completed_at":"2026-10-21T07:51:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-21T07:59:45.683396948Z","timing_metadata_checked_at":"2026-10-21T07:59:45.683396948Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-21T07:59:45.685931034Z","region_metadata_checked_at":"2026-10-21T07:59:45.685931034Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112777231670448128","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-21T05:25:20Z","started_at":"2026-10-21T05:25:20Z","completed_at":"2026-10-21T07:48:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-21T07:59:45.664591155Z","timing_metadata_checked_at":"2026-10-21T07:59:45.664591155Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-21T07:59:45.66727994Z","region_metadata_checked_at":"2026-10-21T07:59:45.66727994Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112776146905993216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-21T05:21:01Z","started_at":"2026-10-21T05:21:01Z","completed_at":"2026-10-21T07:49:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-21T07:59:45.643693574Z","timing_metadata_checked_at":"2026-10-21T07:59:45.643693574Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-21T07:59:45.647957649Z","region_metadata_checked_at":"2026-10-21T07:59:45.647957649Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2112725597401649152","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-21T02:00:09Z","started_at":"2026-10-21T02:00:09Z","completed_at":"2026-10-21T04:11:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-21T04:30:51.770658235Z","timing_metadata_checked_at":"2026-10-21T04:30:51.770658235Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-21T04:30:51.773244528Z","region_metadata_checked_at":"2026-10-21T04:30:51.773244528Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2112718416631566336","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-21T01:31:37Z","started_at":"2026-10-21T01:31:37Z","completed_at":"2026-10-21T04:12:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-21T04:30:51.549182965Z","timing_metadata_checked_at":"2026-10-21T04:30:51.549182965Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-21T04:30:51.552541656Z","region_metadata_checked_at":"2026-10-21T04:30:51.552541656Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}