{"meta":{"date":"2026-09-30","timezone":"UTC","generated_at":"2026-10-29T23:41:02Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":5,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":5,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T23:00:28Z","started_at":"2026-09-30T23:00:28Z","completed_at":"2026-10-01T00:34:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-01T00:36:09.415562976Z","timing_metadata_checked_at":"2026-10-01T00:36:09.415562976Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-01T00:36:09.418072268Z","region_metadata_checked_at":"2026-10-01T00:36:09.418072268Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"250567d2dad0ec032f94249d9bcf7d29c31828244fc119b5b97a3a0fccce9c9b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"297ae9530982a4810dfc39b4017f6105e35c27ea06de54d22f389e155e41a619","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-srbb2hbvsczt/cluster-nightly-5-1-vxb9dm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1af40\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-vxb9dm: failed waiting for cluster=\"cluster-nightly-5-1-vxb9dm\" in resourcegroup=\"rg-nightly-5-1-srbb2hbvsczt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"name\": \"fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T23:26:50.189510673Z\",\n      \"endTime\": \"2026-09-30T23:55:57.188151115Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-105558 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-srbb2hbvsczt/cluster-nightly-5-1-vxb9dm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1af40\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-vxb9dm: failed waiting for cluster=\"cluster-nightly-5-1-vxb9dm\" in resourcegroup=\"rg-nightly-5-1-srbb2hbvsczt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"name\": \"fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T23:26:50.189510673Z\",\n      \"endTime\": \"2026-09-30T23:55:57.188151115Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-105558 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bdaff9adc205561255863f7f3862c4a4a3cdfd05505c7e10e9a53b9e2959cba5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4a8ea4ca61d811f2d283cae7586304bb87d7a7bfe6bf5fc3e9a82ebe6229301e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"754bee22198c76ca5ecb7193f9b82365056113e3559f87b518bbbd3678c80bd6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014907c0\u003e: \n    failed to create NodePool np-prg87r: failed starting nodepool creation \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\" in resourcegroup=\"rg-nightly-4-20-kjrkpmc2vcnb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kjrkpmc2vcnb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-prg87r/nodePools/np-prg87r\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-014938\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014907c0\u003e: \n    failed to create NodePool np-prg87r: failed starting nodepool creation \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\" in resourcegroup=\"rg-nightly-4-20-kjrkpmc2vcnb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kjrkpmc2vcnb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-prg87r/nodePools/np-prg87r\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-014938\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7570604bafa6565b58803f786890e718248e56e844f7447e4608f8464000e8f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"08174c295f58ea65eae7d6210ad569d41e3b1cdc0b0a84a4e29c556f000b91f6","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00154c140\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-49bq5x\" resourcegroup=\"rg-np-dg-minor-49bq5x-7f6vt9z24c7x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-49bq5x-7f6vt9z24c7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-49bq5x/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00154c140\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-49bq5x\" resourcegroup=\"rg-np-dg-minor-49bq5x-7f6vt9z24c7x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-49bq5x-7f6vt9z24c7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-49bq5x/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8b5dbf275e1e89f385dbef51b011d8fb20efbc788ea7bbd6fead6581e08ab58d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"a9ee924e57f9ab9bbddd25402eaa20f29c4967eedecb92e4e71d8d0c06ef84c2","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b92100\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-qglcmw\" in resourcegroup=\"rg-np-version-upgrade-qglcmw-2xxcc62c56zb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-qglcmw-2xxcc62c56zb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-qglcmw/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b92100\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-qglcmw\" in resourcegroup=\"rg-np-version-upgrade-qglcmw-2xxcc62c56zb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-qglcmw-2xxcc62c56zb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-qglcmw/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a63905cc2169d208d54429ce41641ab7ea80696a3d8afea808e54ff74867de7a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","failed_at":"e2e","signature_id":"b7ec09633e70cfe3b7246a6bc21475b497dc8e51890bc8b252d77a2143ce919b","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d74240\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-qlzv6l\" in resourcegroup=\"rg-np-skip-minor-qlzv6l-d9kp5xhmf6nr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-qlzv6l-d9kp5xhmf6nr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-qlzv6l/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d74240\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-qlzv6l\" in resourcegroup=\"rg-np-skip-minor-qlzv6l-d9kp5xhmf6nr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-qlzv6l-d9kp5xhmf6nr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-qlzv6l/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","bdaff9adc205561255863f7f3862c4a4a3cdfd05505c7e10e9a53b9e2959cba5"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T15:51:35Z","started_at":"2026-09-30T15:51:35Z","completed_at":"2026-09-30T18:08:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T18:29:07.732164822Z","timing_metadata_checked_at":"2026-09-30T18:29:07.732164822Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T18:29:07.734945814Z","region_metadata_checked_at":"2026-09-30T18:29:07.734945814Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1f75451f0f3fa7c6b2d4b31a00b4ff21302c48497c25f2fe7aebb0575fd2b1fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","failed_at":"e2e","signature_id":"3933609f5fce2e295de9ab5e9c2e68be690e233f3f4534aede4c689e2581a030","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000119c40\u003e: \n    failed to create NodePool np-djrg27: failed waiting for nodepool=\"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\" in resourcegroup=\"rg-candidate-4-20-hwlm5v4jsr9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"name\": \"94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:46:47.162303776Z\",\n      \"endTime\": \"2026-09-30T17:05:53.284012473Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000119c40\u003e: \n    failed to create NodePool np-djrg27: failed waiting for nodepool=\"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\" in resourcegroup=\"rg-candidate-4-20-hwlm5v4jsr9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"name\": \"94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:46:47.162303776Z\",\n      \"endTime\": \"2026-09-30T17:05:53.284012473Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8ab889d7329c55632f3e73c5be3ffc486e2c2589f066b8767007ab496d59c6a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","failed_at":"e2e","signature_id":"a26abfcbde513ec1ccfb520656c877d6a0b6d347a97e0f7cca974af4807b4a2b","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001703c60\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-jmc9c7\" in resourcegroup=\"rg-np-skip-minor-jmc9c7-9dtxqp29vbsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\",\n      \"name\": \"d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:21:41.15243756Z\",\n      \"endTime\": \"2026-09-30T17:40:44.044506032Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-jmc9c7-9dtxqp29vbsr-np-sm-jmc9c7--managed/providers/Microsoft.Compute/virtualMachines/c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-jmc9c7-9dtxqp29vbsr-np-sm-jmc9c7--managed/providers/Microsoft.Compute/virtualMachines/c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001703c60\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-jmc9c7\" in resourcegroup=\"rg-np-skip-minor-jmc9c7-9dtxqp29vbsr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\",\n      \"name\": \"d4c4dc83-d9cb-477b-aaa0-0403e9664a6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:21:41.15243756Z\",\n      \"endTime\": \"2026-09-30T17:40:44.044506032Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-jmc9c7-9dtxqp29vbsr-np-sm-jmc9c7--managed/providers/Microsoft.Compute/virtualMachines/c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-jmc9c7-9dtxqp29vbsr-np-sm-jmc9c7--managed/providers/Microsoft.Compute/virtualMachines/c4f5p4i1c7o2n5j-nps-420-422-c7z6k-ms2xz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dfb27f0ee5b3afdbf71aab9103209d7b0550b3614d5bb16625715b0f4d6b3cb6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","failed_at":"e2e","signature_id":"2de55e74f48dd0f13326c20bf872d192fa878f413d1746046060d6f653450349","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001116780\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-vh7hggp8cw8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\",\n      \"name\": \"eed44f57-1a91-4587-8901-935588d355ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:09:44.769920829Z\",\n      \"endTime\": \"2026-09-30T17:28:44.009160044Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001116780\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-vh7hggp8cw8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\",\n      \"name\": \"eed44f57-1a91-4587-8901-935588d355ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:09:44.769920829Z\",\n      \"endTime\": \"2026-09-30T17:28:44.009160044Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6stx4mwz4j4k\" to finish creating: GET https://management"}},{"row_id":"ee2506ef305781ed5e9ed673355e6af467a7570e9cd2823743c49f2793bcc57b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","failed_at":"e2e","signature_id":"4aed755d2532b13cc80a0d2fcd018700a14d4af99ab0fa4f4dfa6f963040d21e","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015c0ae0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-twtdf7gdskgb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"name\": \"ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:25:28.987424124Z\",\n      \"endTime\": \"2026-09-30T16:44:30.182558055Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015c0ae0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-twtdf7gdskgb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"name\": \"ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:25:28.987424124Z\",\n      \"endTime\": \"2026-09-30T16:44:30.182558055Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fa7353b081c974760137d2173ac430acdca29ab370ea1f5df1e57b6bcd747605","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","failed_at":"e2e","signature_id":"213150af94f11ecc8fc032938950c97f98aae10d4ca8c4c3d158d38b210c0554","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c2500\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-xhfvnf\" in resourcegroup=\"rg-np-noedge-xhfvnf-d6cdpdqmxzz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"name\": \"79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:42.49451817Z\",\n      \"endTime\": \"2026-09-30T17:39:43.993225307Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c2500\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-xhfvnf\" in resourcegroup=\"rg-np-noedge-xhfvnf-d6cdpdqmxzz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"name\": \"79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:42.49451817Z\",\n      \"endTime\": \"2026-09-30T17:39:43.993225307Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T12:10:26Z","started_at":"2026-09-30T12:10:26Z","completed_at":"2026-09-30T14:28:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T14:48:05.156360818Z","timing_metadata_checked_at":"2026-09-30T14:48:05.156360818Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T14:48:05.163389396Z","region_metadata_checked_at":"2026-09-30T14:48:05.163389396Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"5631922113930af9bf7334768a4ca83318a4bb52715230b6644d21b5d83e740e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","failed_at":"e2e","signature_id":"06f24328299a032f056108312696270e080fc953f45840834663f37907dc3db6","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00148e9c0\u003e: \n    resource group \"rg-zstream-upgrade-5-0-l6m7gzggxh5z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-mrptx7\" in resourcegroup=\"rg-zstream-upgrade-5-0-l6m7gzggxh5z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"name\": \"d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T13:28:47.89996524Z\",\n      \"endTime\": \"2026-09-30T13:52:47.080528249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5bj2pp5rm42moqm553gsgpuq5afnl4 still exists (deletion dispatched at 2026-09-30T13:28:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00148e9c0\u003e: \n    resource group \"rg-zstream-upgrade-5-0-l6m7gzggxh5z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-mrptx7\" in resourcegroup=\"rg-zstream-upgrade-5-0-l6m7gzggxh5z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"name\": \"d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T13:28:47.89996524Z\",\n      \"endTime\": \"2026-09-30T13:52:47.080528249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5bj2pp5rm42moqm553gsgpuq5afnl4 still exists (deletion dispatched at 2026-09-30T13:28:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8e372e8cd22844b25567b8936aae520315cefe944c8204b6d4b9f58401136e31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","failed_at":"e2e","signature_id":"ba33ade4a966c1de67f013ea1140c2c38fb60d9c849af3a44a8bb8f1d3d3e42b","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ed920\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-rkbcqh\" in resourcegroup=\"rg-np-skip-minor-rkbcqh-h7clvkbvfrzv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\",\n      \"name\": \"0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T13:38:48.671242236Z\",\n      \"endTime\": \"2026-09-30T13:57:57.057793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-rkbcqh-h7clvkbvfrzv-np-sm-rkbcqh--managed/providers/Microsoft.Compute/virtualMachines/i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-rkbcqh-h7clvkbvfrzv-np-sm-rkbcqh--managed/providers/Microsoft.Compute/virtualMachines/i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.40\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009ed920\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-rkbcqh\" in resourcegroup=\"rg-np-skip-minor-rkbcqh-h7clvkbvfrzv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\",\n      \"name\": \"0e29588c-4a53-4b7a-8ca9-501b18d6f6cb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T13:38:48.671242236Z\",\n      \"endTime\": \"2026-09-30T13:57:57.057793Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-rkbcqh-h7clvkbvfrzv-np-sm-rkbcqh--managed/providers/Microsoft.Compute/virtualMachines/i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-rkbcqh-h7clvkbvfrzv-np-sm-rkbcqh--managed/providers/Microsoft.Compute/virtualMachines/i3m1n5y2h7p6d9x-nps-420-422-mhgzh-94492 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"91698a44aad5ca32c424df508296f3f679852764e7f79c96c1e59cdf00596e29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","failed_at":"e2e","signature_id":"cda87c02afe385de5c6e530de6be078040d84e3fa8634ad1d769024158056676","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c62580\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4ldsbj98fhm7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b038c9-d01f-4cce-8da4-821ff1fad693\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b038c9-d01f-4cce-8da4-821ff1fad693\",\n      \"name\": \"d5b038c9-d01f-4cce-8da4-821ff1fad693\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:41:58.878961581Z\",\n      \"endTime\": \"2026-09-30T13:01:07.023295945Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4ldsbj98fhm7--managed/providers/Microsoft.Compute/virtualMachines/p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4ldsbj98fhm7--managed/providers/Microsoft.Compute/virtualMachines/p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c62580\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4ldsbj98fhm7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b038c9-d01f-4cce-8da4-821ff1fad693\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5b038c9-d01f-4cce-8da4-821ff1fad693\",\n      \"name\": \"d5b038c9-d01f-4cce-8da4-821ff1fad693\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T12:41:58.878961581Z\",\n      \"endTime\": \"2026-09-30T13:01:07.023295945Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4ldsbj98fhm7--managed/providers/Microsoft.Compute/virtualMachines/p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-4ldsbj98fhm7--managed/providers/Microsoft.Compute/virtualMachines/p9k9r7n9n9q9i7t-np-1-kxfwv-jwz82 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"939f8c5af8420a6aa6e291389f9e8135d165bea7a857a4cd7eb1733e587fdcc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","failed_at":"e2e","signature_id":"b859706588c59cf13ba01690598c1e296b21c66674acdd2387e5df98da83bb15","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd79a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ldfdqgv9q9rv\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd79a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ldfdqgv9q9rv\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83447d3d19023b63c00022eba710d55e1a0090317aba9cb3e48328dc0b521261","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ldfdqgv9q9rv\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","83447d3d19023b63c00022eba710d55e1a0090317aba9cb3e48328dc0b521261","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105187694681264128","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T06:47:13Z","started_at":"2026-09-30T06:47:13Z","completed_at":"2026-09-30T09:01:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T09:11:41.469641612Z","timing_metadata_checked_at":"2026-09-30T09:11:41.469641612Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T09:11:41.472457204Z","region_metadata_checked_at":"2026-09-30T09:11:41.472457204Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4bf1d48fc5aa1eeeed8e1e4cef061704ad02a02b051326fdfafa07cb35cf2eb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105187694681264128","occurred_at":"2026-09-30T06:47:13Z","failed_at":"e2e","signature_id":"e3db6908a51e323a629ef61d2f3ca60b3da46a872c365d1cca8d13ce3880374f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a8a4b0\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-b6jrz8\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"name\": \"8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:07:19.758629115Z\",\n      \"endTime\": \"2026-09-30T08:31:25.87901072Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56tcneq332c0ht4fv2g22jot5kdluc still exists (deletion dispatched at 2026-09-30T08:07:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a8a4b0\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-b6jrz8\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"name\": \"8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:07:19.758629115Z\",\n      \"endTime\": \"2026-09-30T08:31:25.87901072Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56tcneq332c0ht4fv2g22jot5kdluc still exists (deletion dispatched at 2026-09-30T08:07:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b1d3cd1631f041b5259319fed4389bb786eac41289d1c10810e2c1ed7d395fae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105187694681264128","occurred_at":"2026-09-30T06:47:13Z","failed_at":"e2e","signature_id":"793343d9f2198c07b4a1defe40164b7665fed5f9625606d1878699fdfd9f02db","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0013950a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6stx4mwz4j4k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"name\": \"626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:05:30.277326796Z\",\n      \"endTime\": \"2026-09-30T08:24:30.136180053Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0013950a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6stx4mwz4j4k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"name\": \"626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:05:30.277326796Z\",\n      \"endTime\": \"2026-09-30T08:24:30.136180053Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6stx4mwz4j4k\" to finish creating: GET https://management"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-30T02:00:27Z","started_at":"2026-09-30T02:00:27Z","completed_at":"2026-09-30T03:58:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T04:15:05.778248446Z","timing_metadata_checked_at":"2026-09-30T04:15:05.778248446Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T04:15:05.781206637Z","region_metadata_checked_at":"2026-09-30T04:15:05.781206637Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"137960408ac2fe9cc2e1e0f2be91572b0d3eda1c1c26a596515cd4481dac7a09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"5e2362cdd385e535c236eb25d2af3bedec9c45598cf42305119a1c2199da01c0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lssckk\" for cluster \"cluster-candidate-4-22-lssckk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0d380\u003e: \n    failed to create NodePool np-lssckk: failed waiting for nodepool=\"np-lssckk\" for cluster \"cluster-candidate-4-22-lssckk\" in resourcegroup=\"rg-candidate-4-22-bqwkx6ssnq7c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b6149906-5b1c-417e-9e9c-3b2c8a352da8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b6149906-5b1c-417e-9e9c-3b2c8a352da8\",\n      \"name\": \"b6149906-5b1c-417e-9e9c-3b2c8a352da8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:38:56.070433953Z\",\n      \"endTime\": \"2026-09-30T02:57:58.4595049Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-bqwkx6ssnq7c-candidate-lssckk--managed/providers/Microsoft.Compute/virtualMachines/e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-bqwkx6ssnq7c-candidate-lssckk--managed/providers/Microsoft.Compute/virtualMachines/e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lssckk\" for cluster \"cluster-candidate-4-22-lssckk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0d380\u003e: \n    failed to create NodePool np-lssckk: failed waiting for nodepool=\"np-lssckk\" for cluster \"cluster-candidate-4-22-lssckk\" in resourcegroup=\"rg-candidate-4-22-bqwkx6ssnq7c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b6149906-5b1c-417e-9e9c-3b2c8a352da8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b6149906-5b1c-417e-9e9c-3b2c8a352da8\",\n      \"name\": \"b6149906-5b1c-417e-9e9c-3b2c8a352da8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:38:56.070433953Z\",\n      \"endTime\": \"2026-09-30T02:57:58.4595049Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-bqwkx6ssnq7c-candidate-lssckk--managed/providers/Microsoft.Compute/virtualMachines/e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-bqwkx6ssnq7c-candidate-lssckk--managed/providers/Microsoft.Compute/virtualMachines/e6g5l0a6t3i7x6g-np-lssckk-rm7bq-b6c64 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"287863c149517e9259d41fd5ddb52b1ac7fe47bd912cd1a7f835afcf21754c9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"d88867851ef4d43076446e92a3c85889999fd11a09b6284afc83108a89d24f58","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001035640\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4kv8jbqzgm6h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0011df08-d47e-405c-b953-36672f501c27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0011df08-d47e-405c-b953-36672f501c27\",\n      \"name\": \"0011df08-d47e-405c-b953-36672f501c27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T03:17:50.983140083Z\",\n      \"endTime\": \"2026-09-30T03:36:56.457139595Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-4kv8jbqzgm6h--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-4kv8jbqzgm6h--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001035640\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-4kv8jbqzgm6h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0011df08-d47e-405c-b953-36672f501c27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0011df08-d47e-405c-b953-36672f501c27\",\n      \"name\": \"0011df08-d47e-405c-b953-36672f501c27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T03:17:50.983140083Z\",\n      \"endTime\": \"2026-09-30T03:36:56.457139595Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-4kv8jbqzgm6h--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-4kv8jbqzgm6h--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-lm6c8-swzpr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c029d77569b7cc39a439f01d39feca6655374b07db10ca40bcc38570ac2412e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"f280960e298b057c6eef0496e65821ec4103a9ef2176aedafe051906baf460a7","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008827a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-tb59h9fbmsnp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69cf1894-8af7-4f4d-985a-7d39b5ffb985\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69cf1894-8af7-4f4d-985a-7d39b5ffb985\",\n      \"name\": \"69cf1894-8af7-4f4d-985a-7d39b5ffb985\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:15:06.409498278Z\",\n      \"endTime\": \"2026-09-30T02:34:10.118150935Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tb59h9fbmsnp--managed/providers/Microsoft.Compute/virtualMachines/o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tb59h9fbmsnp--managed/providers/Microsoft.Compute/virtualMachines/o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008827a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-tb59h9fbmsnp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69cf1894-8af7-4f4d-985a-7d39b5ffb985\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/69cf1894-8af7-4f4d-985a-7d39b5ffb985\",\n      \"name\": \"69cf1894-8af7-4f4d-985a-7d39b5ffb985\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:15:06.409498278Z\",\n      \"endTime\": \"2026-09-30T02:34:10.118150935Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tb59h9fbmsnp--managed/providers/Microsoft.Compute/virtualMachines/o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tb59h9fbmsnp--managed/providers/Microsoft.Compute/virtualMachines/o2v5c2m5n1d2o2y-np-1-dbbj9-65tmd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dafb1ae9739417d29a65d4fcca8a79d4531b8e7c78aaa5d616c471462c06b6f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"959da02202a889de6291751a794c7fb5d01b45f09a2d70f1fa8fe082a480a903","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016d54b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016d54b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"dd046f4f0751e6475d45e665396196320d5fe6e336b8818055e647f96f904f2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","failed_at":"e2e","signature_id":"d4afdcb13b7abb84b80ed486bbab0e753e39b3cf0759b719778c721a28c9e54b","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c25d80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-jtns6kb9n72s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13d22eca-d7af-43b2-9ab7-e630c81c0521\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13d22eca-d7af-43b2-9ab7-e630c81c0521\",\n      \"name\": \"13d22eca-d7af-43b2-9ab7-e630c81c0521\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:16:05.748001285Z\",\n      \"endTime\": \"2026-09-30T02:35:10.119119459Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-jtns6kb9n72s--managed/providers/Microsoft.Compute/virtualMachines/q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-jtns6kb9n72s--managed/providers/Microsoft.Compute/virtualMachines/q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c25d80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-jtns6kb9n72s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13d22eca-d7af-43b2-9ab7-e630c81c0521\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13d22eca-d7af-43b2-9ab7-e630c81c0521\",\n      \"name\": \"13d22eca-d7af-43b2-9ab7-e630c81c0521\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:16:05.748001285Z\",\n      \"endTime\": \"2026-09-30T02:35:10.119119459Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-jtns6kb9n72s--managed/providers/Microsoft.Compute/virtualMachines/q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-jtns6kb9n72s--managed/providers/Microsoft.Compute/virtualMachines/q5y5f0n3v7s7d3l-np-1-c6z9g-tg2mg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}