{"meta":{"date":"2026-09-26","timezone":"UTC","generated_at":"2026-10-30T03:44:47Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":8,"failed_runs":8,"runs_with_occurrences":8,"runs_with_failure_pattern_matches":8,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-26T23:00:17Z","started_at":"2026-09-26T23:00:17Z","completed_at":"2026-09-27T00:59:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-27T01:04:48.992888877Z","timing_metadata_checked_at":"2026-09-27T01:04:48.992888877Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-27T01:04:48.995893067Z","region_metadata_checked_at":"2026-09-27T01:04:48.995893067Z"},"failed_at":["e2e"],"failed_test_count":17,"occurrences":[{"row_id":"0d3e7551da5637e4829cedb350a15593cdb2a1adee53961ac5419365306cff3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"f05f1056654f1d3969661b5157a3329323ee8a790487854e6d8e017502186852","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-vg7tn9\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f98ae0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-vg7tn9: failed waiting for cluster=\"cp-ystream-upgrade-5-0-vg7tn9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-rxgcfxnprf4k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4dfb91f8-fcd3-478b-b03f-ad43d9671cdb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4dfb91f8-fcd3-478b-b03f-ad43d9671cdb\",\n \"name\": \"4dfb91f8-fcd3-478b-b03f-ad43d9671cdb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:49:36.517184321Z\",\n \"endTime\": \"2026-09-27T00:18:40.854855035Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-vg7tn9\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f98ae0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-vg7tn9: failed waiting for cluster=\"cp-ystream-upgrade-5-0-vg7tn9\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-rxgcfxnprf4k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4dfb91f8-fcd3-478b-b03f-ad43d9671cdb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4dfb91f8-fcd3-478b-b03f-ad43d9671cdb\",\n \"name\": \"4dfb91f8-fcd3-478b-b03f-ad43d9671cdb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:49:36.517184321Z\",\n \"endTime\": \"2026-09-27T00:18:40.854855035Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0dc9b9ed5aaaf4decaf95a9ad1e948fdf5f1d9962ded4e5fbd3ca2a56061a90d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"0851a1986905a5b422808848fdfdf27cfb0c40343e7c8b366a189c08d2b9281b","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001c13890\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-xslh5ql6dz6k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001c13890\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-xslh5ql6dz6k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"17ee738659507dba1f7d1affa976cc33ad9a86c1b7930e10dec924e747618c06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"ed12e18092a6cb363796b8a2d7f31ebc10c5f1a6cfc32b6f300e750016e2970d","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001207440\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-p68sqdmczpdg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001207440\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-p68sqdmczpdg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2efe7ceeb4f93366fce8a6e626436eb9f752c30dd7601b410dc98a47af9d5e76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"4540928969b272dea9930016b863cc4715cd1070f913dd0d298a5c42ca07687e","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000afa620\u003e: \n failed to create HCP cluster np-dg-minor-6vv4rp: failed waiting for cluster=\"np-dg-minor-6vv4rp\" in resourcegroup=\"rg-np-dg-minor-6vv4rp-259t47rttggq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66c0cba9-056f-4636-8679-f28ce98cdca9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66c0cba9-056f-4636-8679-f28ce98cdca9\",\n \"name\": \"66c0cba9-056f-4636-8679-f28ce98cdca9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:05:12.968666607Z\",\n \"endTime\": \"2026-09-27T00:34:17.912617084Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000afa620\u003e: \n failed to create HCP cluster np-dg-minor-6vv4rp: failed waiting for cluster=\"np-dg-minor-6vv4rp\" in resourcegroup=\"rg-np-dg-minor-6vv4rp-259t47rttggq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66c0cba9-056f-4636-8679-f28ce98cdca9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66c0cba9-056f-4636-8679-f28ce98cdca9\",\n \"name\": \"66c0cba9-056f-4636-8679-f28ce98cdca9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:05:12.968666607Z\",\n \"endTime\": \"2026-09-27T00:34:17.912617084Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3049c30f5d721a302e2358ed96fcbee40b746cd6f42c6bcdcc976d9afcda81ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"2ba467d1a14629feb97a61c1ba7852b7404d2206ddc7f3762c7e73e411f2a02b","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-f6zhb5 at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011b080\u003e: \n failed to create HCP cluster np-skip-minor-f6zhb5: failed waiting for cluster=\"np-skip-minor-f6zhb5\" in resourcegroup=\"rg-np-skip-minor-f6zhb5-mltd4bpn2ng8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\",\n \"name\": \"b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:03:26.334155746Z\",\n \"endTime\": \"2026-09-27T00:32:27.905092762Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-f6zhb5 at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011b080\u003e: \n failed to create HCP cluster np-skip-minor-f6zhb5: failed waiting for cluster=\"np-skip-minor-f6zhb5\" in resourcegroup=\"rg-np-skip-minor-f6zhb5-mltd4bpn2ng8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\",\n \"name\": \"b3d18d82-d0f9-496a-97bb-ff0fdefcdec6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-27T00:03:26.334155746Z\",\n \"endTime\": \"2026-09-27T00:32:27.905092762Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3519af81860dae3af3a64ba5a0bef3feedf291f471690a437f2bb1451909df8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"83c3157c23d8aad0c7c51357004cced7cba545ae6f2dfec2d5c5e89d40ae1156","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-v9gzmf\" for cluster \"cluster-nightly-4-20-v9gzmf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ad720\u003e: \n failed to create NodePool np-v9gzmf: failed starting nodepool creation \"np-v9gzmf\" for cluster \"cluster-nightly-4-20-v9gzmf\" in resourcegroup=\"rg-nightly-4-20-lq47jh9j7pkp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-lq47jh9j7pkp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-v9gzmf/nodePools/np-v9gzmf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-v9gzmf\" for cluster \"cluster-nightly-4-20-v9gzmf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016ad720\u003e: \n failed to create NodePool np-v9gzmf: failed starting nodepool creation \"np-v9gzmf\" for cluster \"cluster-nightly-4-20-v9gzmf\" in resourcegroup=\"rg-nightly-4-20-lq47jh9j7pkp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-lq47jh9j7pkp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-v9gzmf/nodePools/np-v9gzmf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5da9e392013a18f7c4c239c92a8d4371de62676d596c2cea3c1aa77bea250ce7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"db9eec2083329b01da8d2cda46e73d2a9ef0942d7ceae885ce0bc1271c38f74e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001b9f6e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-5phhxmkf5cjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001b9f6e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-5phhxmkf5cjv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"65d0f22a9c6c3e9e780fb71adf24bcd950f8cda9953318c0dfad0ebbf79d2920","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"7893b4af94afe3c0281e1d0d5e4a28b92a42f2a0ba01657f8c46a66791bde30e","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005e2bc0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-czbz85\" in resourcegroup=\"rg-np-version-upgrade-czbz85-h7z4wz569lk2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-czbz85-h7z4wz569lk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-czbz85/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005e2bc0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-czbz85\" in resourcegroup=\"rg-np-version-upgrade-czbz85-h7z4wz569lk2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-czbz85-h7z4wz569lk2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-czbz85/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7432a9dae86a91b6f6a5f77a9446f3eee5d60959f4d1ae0bfee13483c2743381","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"ad2672a3362fafb9d20fe759bb8cd0e50a98d6e39bb8af803f5a3851f16a2e13","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001849320\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7h9j6j7d894p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001849320\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7h9j6j7d894p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7b691ac30e1208c7f3e62dc708db96b4e3ce22d34479e6d45022685cdaea18eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"0fb83beb0f702d994cf5a20ec546309e606192d5d552ebb8972f4ca02a52a5a7","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00141cf78\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-09-24-033657, targetMinor=4.22.0-0.nightly-multi-2026-07-08-192436) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-07-08-192436\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00141cf78\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-09-24-033657, targetMinor=4.22.0-0.nightly-multi-2026-07-08-192436) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-07-08-192436\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"7f0d79b3deb6f19f1e0b7071f2ab2a995599bd809795cca13701077f2e95cdca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"838796bcc7d4a7577ada2de39969a525ff7d7bafc0d94b7082c7538d5eda55e6","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014f6150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-zmbfdjmggw6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014f6150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-zmbfdjmggw6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8d58b646861e104167681629613d882de113c84e3ddbd26c11d142d0993e1f36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"11fafce13cda39efdbda8cbfbabf4f5aa9d728b01c3b8cc3fd705a8eb948f07f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-tdqnbhv945ln/cluster-nightly-4-22-hwtgkf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000936960\u003e: \n failed to create HCP cluster cluster-nightly-4-22-hwtgkf: failed waiting for cluster=\"cluster-nightly-4-22-hwtgkf\" in resourcegroup=\"rg-nightly-4-22-tdqnbhv945ln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c8f5770-a2ce-4214-855c-9d70572faf14\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c8f5770-a2ce-4214-855c-9d70572faf14\",\n \"name\": \"2c8f5770-a2ce-4214-855c-9d70572faf14\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:43:51.36895754Z\",\n \"endTime\": \"2026-09-27T00:13:00.824555826Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-tdqnbhv945ln/cluster-nightly-4-22-hwtgkf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000936960\u003e: \n failed to create HCP cluster cluster-nightly-4-22-hwtgkf: failed waiting for cluster=\"cluster-nightly-4-22-hwtgkf\" in resourcegroup=\"rg-nightly-4-22-tdqnbhv945ln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c8f5770-a2ce-4214-855c-9d70572faf14\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2c8f5770-a2ce-4214-855c-9d70572faf14\",\n \"name\": \"2c8f5770-a2ce-4214-855c-9d70572faf14\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:43:51.36895754Z\",\n \"endTime\": \"2026-09-27T00:13:00.824555826Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a946f14c7ec950707dac463d05210eee6f5aac22963726d15af443b541d2a72d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"089da5e62c30a2a9e3d1a6245182e71a4642a38a698b3b82c0cd96b9913dc8e0","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011a380\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-kb4s8cpxl5xw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66b0c680-744e-463d-83fe-32a6b03b0059\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66b0c680-744e-463d-83fe-32a6b03b0059\",\n \"name\": \"66b0c680-744e-463d-83fe-32a6b03b0059\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:30:24.096246355Z\",\n \"endTime\": \"2026-09-26T23:49:30.686897638Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m1h7k2a6w9l9r1k-np-1-hqxhh-kj929: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-kb4s8cpxl5xw--managed/providers/Microsoft.Compute/virtualMachines/m1h7k2a6w9l9r1k-np-1-hqxhh-kj929 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m1h7k2a6w9l9r1k-np-1-hqxhh-kj929: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-kb4s8cpxl5xw--managed/providers/Microsoft.Compute/virtualMachines/m1h7k2a6w9l9r1k-np-1-hqxhh-kj929 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011a380\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-kb4s8cpxl5xw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66b0c680-744e-463d-83fe-32a6b03b0059\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/66b0c680-744e-463d-83fe-32a6b03b0059\",\n \"name\": \"66b0c680-744e-463d-83fe-32a6b03b0059\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:30:24.096246355Z\",\n \"endTime\": \"2026-09-26T23:49:30.686897638Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m1h7k2a6w9l9r1k-np-1-hqxhh-kj929: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-kb4s8cpxl5xw--managed/providers/Microsoft.Compute/virtualMachines/m1h7k2a6w9l9r1k-np-1-hqxhh-kj929 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m1h7k2a6w9l9r1k-np-1-hqxhh-kj929: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-kb4s8cpxl5xw--managed/providers/Microsoft.Compute/virtualMachines/m1h7k2a6w9l9r1k-np-1-hqxhh-kj929 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ad5dff1fad518de47a2b401e1ea966e141ede16c4c6ecc3aeddb191242cc2022","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"d68ec7aad4b4702fa2feadc626a6b2e344a7de36646823c29820e973bf0cd18b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-8f7dzbcrbw7p/cluster-nightly-5-1-b2l6kn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f37180\u003e: \n failed to create HCP cluster cluster-nightly-5-1-b2l6kn: failed waiting for cluster=\"cluster-nightly-5-1-b2l6kn\" in resourcegroup=\"rg-nightly-5-1-8f7dzbcrbw7p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e80d0504-5f18-4937-9363-cfe3fda28970\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e80d0504-5f18-4937-9363-cfe3fda28970\",\n \"name\": \"e80d0504-5f18-4937-9363-cfe3fda28970\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:46:02.695503366Z\",\n \"endTime\": \"2026-09-27T00:15:10.822999259Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-8f7dzbcrbw7p/cluster-nightly-5-1-b2l6kn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f37180\u003e: \n failed to create HCP cluster cluster-nightly-5-1-b2l6kn: failed waiting for cluster=\"cluster-nightly-5-1-b2l6kn\" in resourcegroup=\"rg-nightly-5-1-8f7dzbcrbw7p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e80d0504-5f18-4937-9363-cfe3fda28970\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e80d0504-5f18-4937-9363-cfe3fda28970\",\n \"name\": \"e80d0504-5f18-4937-9363-cfe3fda28970\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:46:02.695503366Z\",\n \"endTime\": \"2026-09-27T00:15:10.822999259Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"afa735e5b0fd4d7910becac41d78fdc74c6f652fafd23add1d51c416decf8caa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"1b6a70986e38df8901401ab55acb5556c4468c54133387e22f1e45a710ed2281","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0017240c0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0017240c0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"b19348c190395108dd96fa0c19d1b5a97123980cefcd4a0162683f9939cacc14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"f9d15553519d6f900df80d9635a99e16a0d600a336f2c5c4d29422be77ce186f","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001907188\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hpx4nnz688cq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5c2ff73-ec69-46ee-a311-273a07e5930d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5c2ff73-ec69-46ee-a311-273a07e5930d\",\n \"name\": \"d5c2ff73-ec69-46ee-a311-273a07e5930d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:33:38.153702381Z\",\n \"endTime\": \"2026-09-26T23:52:40.648818601Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cdrhx-29bn6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cdrhx-29bn6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n encountered errors downloading console logs:\n - VM \"cilium-cluster-cilium-np-cdrhx-29bn6\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-cdrhx-29bn6\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6/retrieveBootDiagnosticsData\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: OperationNotAllowed\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"OperationNotAllowed\",\n \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-cdrhx-29bn6' since another operation is in progress.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001907188\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-hpx4nnz688cq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5c2ff73-ec69-46ee-a311-273a07e5930d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5c2ff73-ec69-46ee-a311-273a07e5930d\",\n \"name\": \"d5c2ff73-ec69-46ee-a311-273a07e5930d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:33:38.153702381Z\",\n \"endTime\": \"2026-09-26T23:52:40.648818601Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cdrhx-29bn6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cdrhx-29bn6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n encountered errors downloading console logs:\n - VM \"cilium-cluster-cilium-np-cdrhx-29bn6\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-cdrhx-29bn6\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-hpx4nnz688cq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cdrhx-29bn6/retrieveBootDiagnosticsData\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: OperationNotAllowed\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"OperationNotAllowed\",\n \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-cdrhx-29bn6' since another operation is in progress.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cd45c26e64b768cfa63277726bbc10c80939940ccbb95372b16345b70bcf1205","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103983023136444416","occurred_at":"2026-09-26T23:00:17Z","failed_at":"e2e","signature_id":"501206e43f5c4171ac14d29667891202cc96e10a3428f9f639c55ac66c20afba","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-bk2bfx\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00089aa80\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-bk2bfx: failed waiting for cluster=\"cp-ystream-upgrade-4-23-bk2bfx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t8gzs6dzd69p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c52163c-38c9-4f22-9154-361c3d2553ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c52163c-38c9-4f22-9154-361c3d2553ba\",\n \"name\": \"9c52163c-38c9-4f22-9154-361c3d2553ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:49:13.237430377Z\",\n \"endTime\": \"2026-09-27T00:18:20.853759656Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-bk2bfx\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00089aa80\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-bk2bfx: failed waiting for cluster=\"cp-ystream-upgrade-4-23-bk2bfx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-t8gzs6dzd69p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c52163c-38c9-4f22-9154-361c3d2553ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c52163c-38c9-4f22-9154-361c3d2553ba\",\n \"name\": \"9c52163c-38c9-4f22-9154-361c3d2553ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T23:49:13.237430377Z\",\n \"endTime\": \"2026-09-27T00:18:20.853759656Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":17,"failure_pattern_ids":["28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":17,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103956302358122496","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-26T21:14:06Z","started_at":"2026-09-26T21:14:06Z","completed_at":"2026-09-26T23:18:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T23:30:50.127136037Z","timing_metadata_checked_at":"2026-09-26T23:30:50.127136037Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T23:30:50.130604226Z","region_metadata_checked_at":"2026-09-26T23:30:50.130604226Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1722a3f894d69a3d732a65a005c7d43fe74a92bdc88a719df1996c61535fe0c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103956302358122496","occurred_at":"2026-09-26T21:14:06Z","failed_at":"e2e","signature_id":"dbd0f8e48eac3e4cc622d79cb9d2ec8cbadb4a96fb50d05cb0b2554a97288e9e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0028b2360\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-brtk2h8rsjng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc36dfc4-12f6-4dff-948b-9d2f00f4cdab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc36dfc4-12f6-4dff-948b-9d2f00f4cdab\",\n \"name\": \"bc36dfc4-12f6-4dff-948b-9d2f00f4cdab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T21:29:04.025452385Z\",\n \"endTime\": \"2026-09-26T21:48:08.955970797Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-sxr66-dqnpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-brtk2h8rsjng--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-sxr66-dqnpn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-sxr66-dqnpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-brtk2h8rsjng--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-sxr66-dqnpn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n encountered errors downloading console logs:\n - VM \"cilium-cluster-cilium-np-sxr66-dqnpn\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-sxr66-dqnpn\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-brtk2h8rsjng--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-sxr66-dqnpn/retrieveBootDiagnosticsData\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: OperationNotAllowed\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"OperationNotAllowed\",\n \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-sxr66-dqnpn' since another operation is in progress.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0028b2360\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-brtk2h8rsjng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc36dfc4-12f6-4dff-948b-9d2f00f4cdab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bc36dfc4-12f6-4dff-948b-9d2f00f4cdab\",\n \"name\": \"bc36dfc4-12f6-4dff-948b-9d2f00f4cdab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T21:29:04.025452385Z\",\n \"endTime\": \"2026-09-26T21:48:08.955970797Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-sxr66-dqnpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-brtk2h8rsjng--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-sxr66-dqnpn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-sxr66-dqnpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-brtk2h8rsjng--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-sxr66-dqnpn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n encountered errors downloading console logs:\n - VM \"cilium-cluster-cilium-np-sxr66-dqnpn\": failed to retrieve boot diagnostics data for VM \"cilium-cluster-cilium-np-sxr66-dqnpn\" after 3 attempts, last error: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-brtk2h8rsjng--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-sxr66-dqnpn/retrieveBootDiagnosticsData\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: OperationNotAllowed\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"OperationNotAllowed\",\n \"message\": \"Operation 'retrieveBootDiagnosticsData' is not allowed on VM 'cilium-cluster-cilium-np-sxr66-dqnpn' since another operation is in progress.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"84cd93a221d291333c117bfa96be1d34f327c8d97e260a6f309f35aad2ffc014","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103956302358122496","occurred_at":"2026-09-26T21:14:06Z","failed_at":"e2e","signature_id":"4e0a0620c8831c0f951a548fa2513f2279819e6296978d0d4cf8e21c727d4cec","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:598]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000915360\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-94wr55\" in resourcegroup=\"rg-np-downgrade-94wr55-49w8dwsv7jfz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65b789c4-2968-40e8-b87b-ea9ebda5d678\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65b789c4-2968-40e8-b87b-ea9ebda5d678\",\n \"name\": \"65b789c4-2968-40e8-b87b-ea9ebda5d678\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:23:09.106743803Z\",\n \"endTime\": \"2026-09-26T22:42:12.893995625Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-94wr55-49w8dwsv7jfz-np-dg-94wr55--managed/providers/Microsoft.Compute/virtualMachines/s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-94wr55-49w8dwsv7jfz-np-dg-94wr55--managed/providers/Microsoft.Compute/virtualMachines/s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:598]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000915360\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-94wr55\" in resourcegroup=\"rg-np-downgrade-94wr55-49w8dwsv7jfz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65b789c4-2968-40e8-b87b-ea9ebda5d678\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/65b789c4-2968-40e8-b87b-ea9ebda5d678\",\n \"name\": \"65b789c4-2968-40e8-b87b-ea9ebda5d678\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T22:23:09.106743803Z\",\n \"endTime\": \"2026-09-26T22:42:12.893995625Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-94wr55-49w8dwsv7jfz-np-dg-94wr55--managed/providers/Microsoft.Compute/virtualMachines/s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-94wr55-49w8dwsv7jfz-np-dg-94wr55--managed/providers/Microsoft.Compute/virtualMachines/s5n9q0l4z0q6e9r-npdg-4-21-k5pss-hmspx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8b499eacce00b0c947d1656e6aae9db144d0c53d178af0ce92f21e82999c9536","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103956302358122496","occurred_at":"2026-09-26T21:14:06Z","failed_at":"e2e","signature_id":"be9b62e6f592acd1637d57bc1488a35a561ba98109cf2972d839ae6a04fc56ce","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014c6168\u003e: \n resource group \"tls-endpoint-cluster-s8s7nfpcrjx5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-s8s7nfpcrjx5\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n \"name\": \"add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T21:34:10.468425235Z\",\n \"endTime\": \"2026-09-26T21:58:19.111858781Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2ugn3l1bvjlm39njkp5ap0fh8ek88r still exists (deletion dispatched at 2026-09-26T21:34:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014c6168\u003e: \n resource group \"tls-endpoint-cluster-s8s7nfpcrjx5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-s8s7nfpcrjx5\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n \"name\": \"add1f9fb-6a8e-4417-8717-b28b0f76087f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T21:34:10.468425235Z\",\n \"endTime\": \"2026-09-26T21:58:19.111858781Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2ugn3l1bvjlm39njkp5ap0fh8ek88r still exists (deletion dispatched at 2026-09-26T21:34:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103923127544188928","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-26T19:02:17Z","started_at":"2026-09-26T19:02:17Z","completed_at":"2026-09-26T21:13:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T21:16:41.943749186Z","timing_metadata_checked_at":"2026-09-26T21:16:41.943749186Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T21:16:41.946503178Z","region_metadata_checked_at":"2026-09-26T21:16:41.946503178Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9883892e7549f4ad8fe80f395954cf3a07ba7289ee562bfb58263377e3bf0e67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103923127544188928","occurred_at":"2026-09-26T19:02:17Z","failed_at":"e2e","signature_id":"5c06e0d910ddc2acb4d5603c5eadfde618b3dd69052efab39a6a1f1a0a93424d","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001417120\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-hz29dh4r44cz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5d69a1b-6093-4eff-815f-ec78ccfb28e7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5d69a1b-6093-4eff-815f-ec78ccfb28e7\",\n \"name\": \"d5d69a1b-6093-4eff-815f-ec78ccfb28e7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T20:29:03.064278124Z\",\n \"endTime\": \"2026-09-26T20:48:03.684881036Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y2z0t7s5v3i5o8p-np-1-pm27w-q5ghp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-hz29dh4r44cz--managed/providers/Microsoft.Compute/virtualMachines/y2z0t7s5v3i5o8p-np-1-pm27w-q5ghp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y2z0t7s5v3i5o8p-np-1-pm27w-q5ghp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-hz29dh4r44cz--managed/providers/Microsoft.Compute/virtualMachines/y2z0t7s5v3i5o8p-np-1-pm27w-q5ghp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001417120\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-hz29dh4r44cz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5d69a1b-6093-4eff-815f-ec78ccfb28e7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d5d69a1b-6093-4eff-815f-ec78ccfb28e7\",\n \"name\": \"d5d69a1b-6093-4eff-815f-ec78ccfb28e7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T20:29:03.064278124Z\",\n \"endTime\": \"2026-09-26T20:48:03.684881036Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y2z0t7s5v3i5o8p-np-1-pm27w-q5ghp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-hz29dh4r44cz--managed/providers/Microsoft.Compute/virtualMachines/y2z0t7s5v3i5o8p-np-1-pm27w-q5ghp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y2z0t7s5v3i5o8p-np-1-pm27w-q5ghp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-hz29dh4r44cz--managed/providers/Microsoft.Compute/virtualMachines/y2z0t7s5v3i5o8p-np-1-pm27w-q5ghp to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a4fea2a47a0e4cf68ecdbc302c95a58b25a123fdfc8063d9b1a8729b89689628","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103923127544188928","occurred_at":"2026-09-26T19:02:17Z","failed_at":"e2e","signature_id":"a76814b56f15ee47998572c134135162901cee1455aa045f2734242c1a906a9a","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c7e0\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-wqmzcwndlf2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/287407fa-d792-43c8-a3ac-bf8d179a2257\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/287407fa-d792-43c8-a3ac-bf8d179a2257\",\n \"name\": \"287407fa-d792-43c8-a3ac-bf8d179a2257\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T19:30:11.133018346Z\",\n \"endTime\": \"2026-09-26T19:49:20.223545331Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z7b9z5h4y0c0o9n-nodepool-128gib-7dvff-gjh67: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-wqmzcwndlf2p--managed/providers/Microsoft.Compute/virtualMachines/z7b9z5h4y0c0o9n-nodepool-128gib-7dvff-gjh67 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z7b9z5h4y0c0o9n-nodepool-128gib-7dvff-gjh67: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-wqmzcwndlf2p--managed/providers/Microsoft.Compute/virtualMachines/z7b9z5h4y0c0o9n-nodepool-128gib-7dvff-gjh67 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c7e0\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-wqmzcwndlf2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/287407fa-d792-43c8-a3ac-bf8d179a2257\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/287407fa-d792-43c8-a3ac-bf8d179a2257\",\n \"name\": \"287407fa-d792-43c8-a3ac-bf8d179a2257\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T19:30:11.133018346Z\",\n \"endTime\": \"2026-09-26T19:49:20.223545331Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z7b9z5h4y0c0o9n-nodepool-128gib-7dvff-gjh67: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-wqmzcwndlf2p--managed/providers/Microsoft.Compute/virtualMachines/z7b9z5h4y0c0o9n-nodepool-128gib-7dvff-gjh67 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z7b9z5h4y0c0o9n-nodepool-128gib-7dvff-gjh67: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-wqmzcwndlf2p--managed/providers/Microsoft.Compute/virtualMachines/z7b9z5h4y0c0o9n-nodepool-128gib-7dvff-gjh67 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-26T14:41:27Z","started_at":"2026-09-26T14:41:27Z","completed_at":"2026-09-26T16:53:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T17:02:52.794096919Z","timing_metadata_checked_at":"2026-09-26T17:02:52.794096919Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T17:02:52.79712991Z","region_metadata_checked_at":"2026-09-26T17:02:52.79712991Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"5039b7efddd11452d18866ed3eb5b6eac7e4b5758b34c5b786a422f011eaf38b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","failed_at":"e2e","signature_id":"2c1b9ed23893075bc0ec6c4a6b1037e9cca96febebc0b50ef57090534c5cb402","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00107c880\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-xnlxm6q7lcm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ff23c508-1534-45b0-afe1-322eade0805c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ff23c508-1534-45b0-afe1-322eade0805c\",\n \"name\": \"ff23c508-1534-45b0-afe1-322eade0805c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T15:14:28.73622764Z\",\n \"endTime\": \"2026-09-26T15:33:28.526950876Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine t3g2u3v4h9d5k0f-np-1-kr8jm-h74pm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-xnlxm6q7lcm9--managed/providers/Microsoft.Compute/virtualMachines/t3g2u3v4h9d5k0f-np-1-kr8jm-h74pm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine t3g2u3v4h9d5k0f-np-1-kr8jm-h74pm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-xnlxm6q7lcm9--managed/providers/Microsoft.Compute/virtualMachines/t3g2u3v4h9d5k0f-np-1-kr8jm-h74pm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00107c880\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-xnlxm6q7lcm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ff23c508-1534-45b0-afe1-322eade0805c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ff23c508-1534-45b0-afe1-322eade0805c\",\n \"name\": \"ff23c508-1534-45b0-afe1-322eade0805c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T15:14:28.73622764Z\",\n \"endTime\": \"2026-09-26T15:33:28.526950876Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine t3g2u3v4h9d5k0f-np-1-kr8jm-h74pm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-xnlxm6q7lcm9--managed/providers/Microsoft.Compute/virtualMachines/t3g2u3v4h9d5k0f-np-1-kr8jm-h74pm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine t3g2u3v4h9d5k0f-np-1-kr8jm-h74pm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-xnlxm6q7lcm9--managed/providers/Microsoft.Compute/virtualMachines/t3g2u3v4h9d5k0f-np-1-kr8jm-h74pm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"79235d9c487f205ab3a88e90653fc3cee781b247675342a8f7aa867b3a3e0a0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","failed_at":"e2e","signature_id":"7049cd27c77f3381a35c5d814c9d2733d694154d7c87b77c2714762c51e52aaa","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001af6f48\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-xwhrsdh6tmlv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f6fb1ff-b199-44fc-b474-a371b575c211\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f6fb1ff-b199-44fc-b474-a371b575c211\",\n \"name\": \"7f6fb1ff-b199-44fc-b474-a371b575c211\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T15:13:41.359494364Z\",\n \"endTime\": \"2026-09-26T15:32:48.541665354Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-mhn9k-npbsd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-xwhrsdh6tmlv--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-mhn9k-npbsd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-mhn9k-npbsd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-xwhrsdh6tmlv--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-mhn9k-npbsd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001af6f48\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-xwhrsdh6tmlv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f6fb1ff-b199-44fc-b474-a371b575c211\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7f6fb1ff-b199-44fc-b474-a371b575c211\",\n \"name\": \"7f6fb1ff-b199-44fc-b474-a371b575c211\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T15:13:41.359494364Z\",\n \"endTime\": \"2026-09-26T15:32:48.541665354Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-mhn9k-npbsd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-xwhrsdh6tmlv--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-mhn9k-npbsd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-mhn9k-npbsd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-xwhrsdh6tmlv--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-mhn9k-npbsd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"98b5998e3e0131fb9d4a1473a3f4f8525f4725962f9ce61687a2ce35612e8fdd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","failed_at":"e2e","signature_id":"103961703b8016ef8d2dc0e2f58b7bbb98c3574d1dc3c74e67f1d240fdf03492","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a52a80\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-z8x96q\" in resourcegroup=\"rg-np-noedge-z8x96q-2kdtqk7hg4n9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9feb1206-42e5-45be-9e7f-937293863c77\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9feb1206-42e5-45be-9e7f-937293863c77\",\n \"name\": \"9feb1206-42e5-45be-9e7f-937293863c77\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T16:09:21.39874787Z\",\n \"endTime\": \"2026-09-26T16:28:25.629531487Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z6z5b4r3d3c5x2u-npnoedge-4-20-hkdwv-g8gpq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-z8x96q-2kdtqk7hg4n9-np-ne-z8x96q--managed/providers/Microsoft.Compute/virtualMachines/z6z5b4r3d3c5x2u-npnoedge-4-20-hkdwv-g8gpq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z6z5b4r3d3c5x2u-npnoedge-4-20-hkdwv-g8gpq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-z8x96q-2kdtqk7hg4n9-np-ne-z8x96q--managed/providers/Microsoft.Compute/virtualMachines/z6z5b4r3d3c5x2u-npnoedge-4-20-hkdwv-g8gpq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a52a80\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-z8x96q\" in resourcegroup=\"rg-np-noedge-z8x96q-2kdtqk7hg4n9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9feb1206-42e5-45be-9e7f-937293863c77\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9feb1206-42e5-45be-9e7f-937293863c77\",\n \"name\": \"9feb1206-42e5-45be-9e7f-937293863c77\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T16:09:21.39874787Z\",\n \"endTime\": \"2026-09-26T16:28:25.629531487Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z6z5b4r3d3c5x2u-npnoedge-4-20-hkdwv-g8gpq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-z8x96q-2kdtqk7hg4n9-np-ne-z8x96q--managed/providers/Microsoft.Compute/virtualMachines/z6z5b4r3d3c5x2u-npnoedge-4-20-hkdwv-g8gpq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z6z5b4r3d3c5x2u-npnoedge-4-20-hkdwv-g8gpq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-z8x96q-2kdtqk7hg4n9-np-ne-z8x96q--managed/providers/Microsoft.Compute/virtualMachines/z6z5b4r3d3c5x2u-npnoedge-4-20-hkdwv-g8gpq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d68e801190590bee925f6efebeceafc71bc8277964c07d590b864fcf4c5abfeb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","failed_at":"e2e","signature_id":"82368bdfff8d1629c3e37cf188eb5b40952905d54948f637fde213abe6a41563","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d436e0\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-v7prvk\" in resourcegroup=\"rg-np-skip-minor-v7prvk-5b2q5bj8c6pf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85e9d6dd-362f-4b59-9640-d13f74c43941\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85e9d6dd-362f-4b59-9640-d13f74c43941\",\n \"name\": \"85e9d6dd-362f-4b59-9640-d13f74c43941\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T16:07:25.346522094Z\",\n \"endTime\": \"2026-09-26T16:26:25.857169591Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i4x5r0s3e6z4u1b-nps-420-422-r55vg-fj7nb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-v7prvk-5b2q5bj8c6pf-np-sm-v7prvk--managed/providers/Microsoft.Compute/virtualMachines/i4x5r0s3e6z4u1b-nps-420-422-r55vg-fj7nb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i4x5r0s3e6z4u1b-nps-420-422-r55vg-fj7nb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-v7prvk-5b2q5bj8c6pf-np-sm-v7prvk--managed/providers/Microsoft.Compute/virtualMachines/i4x5r0s3e6z4u1b-nps-420-422-r55vg-fj7nb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d436e0\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-v7prvk\" in resourcegroup=\"rg-np-skip-minor-v7prvk-5b2q5bj8c6pf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85e9d6dd-362f-4b59-9640-d13f74c43941\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85e9d6dd-362f-4b59-9640-d13f74c43941\",\n \"name\": \"85e9d6dd-362f-4b59-9640-d13f74c43941\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T16:07:25.346522094Z\",\n \"endTime\": \"2026-09-26T16:26:25.857169591Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i4x5r0s3e6z4u1b-nps-420-422-r55vg-fj7nb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-v7prvk-5b2q5bj8c6pf-np-sm-v7prvk--managed/providers/Microsoft.Compute/virtualMachines/i4x5r0s3e6z4u1b-nps-420-422-r55vg-fj7nb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i4x5r0s3e6z4u1b-nps-420-422-r55vg-fj7nb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-v7prvk-5b2q5bj8c6pf-np-sm-v7prvk--managed/providers/Microsoft.Compute/virtualMachines/i4x5r0s3e6z4u1b-nps-420-422-r55vg-fj7nb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"edcdb43d796c9c7aa9a0ec04e09c54f986ba293b6b85d663bf83f3eead3d4d35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","failed_at":"e2e","signature_id":"28839740e2f9c23000e6b0500d8c0408bf8e17372a61541f2937024db1d36025","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000535720\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mtj55995hz55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ae86850-245b-4016-bb60-b4b035485db0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ae86850-245b-4016-bb60-b4b035485db0\",\n \"name\": \"8ae86850-245b-4016-bb60-b4b035485db0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T15:57:07.959371171Z\",\n \"endTime\": \"2026-09-26T16:16:15.682428941Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine i5e8l8z4l0r7p7c-autoscale-noaz-mdlgl-vsxt8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-mtj55995hz55--managed/providers/Microsoft.Compute/virtualMachines/i5e8l8z4l0r7p7c-autoscale-noaz-mdlgl-vsxt8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine i5e8l8z4l0r7p7c-autoscale-noaz-mdlgl-vsxt8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-mtj55995hz55--managed/providers/Microsoft.Compute/virtualMachines/i5e8l8z4l0r7p7c-autoscale-noaz-mdlgl-vsxt8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000535720\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mtj55995hz55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ae86850-245b-4016-bb60-b4b035485db0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ae86850-245b-4016-bb60-b4b035485db0\",\n \"name\": \"8ae86850-245b-4016-bb60-b4b035485db0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T15:57:07.959371171Z\",\n \"endTime\": \"2026-09-26T16:16:15.682428941Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine i5e8l8z4l0r7p7c-autoscale-noaz-mdlgl-vsxt8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-mtj55995hz55--managed/providers/Microsoft.Compute/virtualMachines/i5e8l8z4l0r7p7c-autoscale-noaz-mdlgl-vsxt8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine i5e8l8z4l0r7p7c-autoscale-noaz-mdlgl-vsxt8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-mtj55995hz55--managed/providers/Microsoft.Compute/virtualMachines/i5e8l8z4l0r7p7c-autoscale-noaz-mdlgl-vsxt8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f3c8f756319bfcdf6cb5836a71997d61909a0e76facc2f22e915d4e00186219a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103857488750186496","occurred_at":"2026-09-26T14:41:27Z","failed_at":"e2e","signature_id":"65253901d9a0188b19f3893d1b92022170a5abe60d3935d41601fb4c5d5f1b5a","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b44360\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-j2mzj6vms9k6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1c9994c-76dc-4ba9-a74c-2e9f6ee89b81\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1c9994c-76dc-4ba9-a74c-2e9f6ee89b81\",\n \"name\": \"e1c9994c-76dc-4ba9-a74c-2e9f6ee89b81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T15:13:01.589549739Z\",\n \"endTime\": \"2026-09-26T15:32:08.767838782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine t6o2m0h2e0w4n7z-arm64-vm-np-1-kfrdk-p9nsm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-j2mzj6vms9k6--managed/providers/Microsoft.Compute/virtualMachines/t6o2m0h2e0w4n7z-arm64-vm-np-1-kfrdk-p9nsm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine t6o2m0h2e0w4n7z-arm64-vm-np-1-kfrdk-p9nsm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-j2mzj6vms9k6--managed/providers/Microsoft.Compute/virtualMachines/t6o2m0h2e0w4n7z-arm64-vm-np-1-kfrdk-p9nsm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b44360\u003e: \n failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-j2mzj6vms9k6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1c9994c-76dc-4ba9-a74c-2e9f6ee89b81\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e1c9994c-76dc-4ba9-a74c-2e9f6ee89b81\",\n \"name\": \"e1c9994c-76dc-4ba9-a74c-2e9f6ee89b81\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T15:13:01.589549739Z\",\n \"endTime\": \"2026-09-26T15:32:08.767838782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine t6o2m0h2e0w4n7z-arm64-vm-np-1-kfrdk-p9nsm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-j2mzj6vms9k6--managed/providers/Microsoft.Compute/virtualMachines/t6o2m0h2e0w4n7z-arm64-vm-np-1-kfrdk-p9nsm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine t6o2m0h2e0w4n7z-arm64-vm-np-1-kfrdk-p9nsm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-j2mzj6vms9k6--managed/providers/Microsoft.Compute/virtualMachines/t6o2m0h2e0w4n7z-arm64-vm-np-1-kfrdk-p9nsm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103778668198236160","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-26T09:28:15Z","started_at":"2026-09-26T09:28:15Z","completed_at":"2026-09-26T14:05:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T14:15:00.107598155Z","timing_metadata_checked_at":"2026-09-26T14:15:00.107598155Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T14:15:00.110262047Z","region_metadata_checked_at":"2026-09-26T14:15:00.110262047Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"23c80984b009c01f2f8e134204347bce3bd8ddfed526950ac2d99d73b4a9f019","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103778668198236160","occurred_at":"2026-09-26T09:28:15Z","failed_at":"e2e","signature_id":"3e311527d14456ac6894554b956c239196c7332831ad87223df28d3ee5ad65c6","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001199c0\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-rzctzfm7wj8d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d62a8c5-db27-49e9-941f-20e83e0e72a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d62a8c5-db27-49e9-941f-20e83e0e72a4\",\n \"name\": \"6d62a8c5-db27-49e9-941f-20e83e0e72a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T10:32:28.860402325Z\",\n \"endTime\": \"2026-09-26T10:51:36.244471948Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-hl4rt-rwrt6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-rzctzfm7wj8d--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-hl4rt-rwrt6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-hl4rt-rwrt6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-rzctzfm7wj8d--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-hl4rt-rwrt6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001199c0\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-rzctzfm7wj8d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d62a8c5-db27-49e9-941f-20e83e0e72a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d62a8c5-db27-49e9-941f-20e83e0e72a4\",\n \"name\": \"6d62a8c5-db27-49e9-941f-20e83e0e72a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T10:32:28.860402325Z\",\n \"endTime\": \"2026-09-26T10:51:36.244471948Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-hl4rt-rwrt6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-rzctzfm7wj8d--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-hl4rt-rwrt6 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-hl4rt-rwrt6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-rzctzfm7wj8d--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-hl4rt-rwrt6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"608365fc9eb61a4c629d5166e1d3f6af97f853174331caca8ae67bc1e460369a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103778668198236160","occurred_at":"2026-09-26T09:28:15Z","failed_at":"e2e","signature_id":"b827dee0a9cbc7d4f17c48bae9a3424d764e85fb3cd960112ea93b2d8250799d","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a00138\u003e: \n resource group \"pullsecret-test-5vdpdhbfbqdk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5vdpdhbfbqdk\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n \"name\": \"82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T10:04:15.110849507Z\",\n \"endTime\": \"2026-09-26T10:28:16.325232898Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdg1r6crhcm349aur60m1j1fcbuh2 still exists (deletion dispatched at 2026-09-26T10:04:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a00138\u003e: \n resource group \"pullsecret-test-5vdpdhbfbqdk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-5vdpdhbfbqdk\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n \"name\": \"82ed94ae-6b5c-42a4-9f90-3280fa2efff4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T10:04:15.110849507Z\",\n \"endTime\": \"2026-09-26T10:28:16.325232898Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdg1r6crhcm349aur60m1j1fcbuh2 still exists (deletion dispatched at 2026-09-26T10:04:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"faea17451cefa717d12e8f3a43f4f2ce139f52fbfd8c39442bd2247badef1de2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103778668198236160","occurred_at":"2026-09-26T09:28:15Z","failed_at":"e2e","signature_id":"66487e377bb5836614b65c245bd64eda43f31dc7d151c0a386ac9d9a8cd2fbe9","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c140c0\u003e: \n resource group \"tls-endpoint-cluster-tlzh4bzgw4zh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-tlzh4bzgw4zh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n \"name\": \"3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T10:04:17.573277188Z\",\n \"endTime\": \"2026-09-26T10:28:26.278499332Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdfqqgc9osqgq67u99r2mbueuta1t still exists (deletion dispatched at 2026-09-26T10:04:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c140c0\u003e: \n resource group \"tls-endpoint-cluster-tlzh4bzgw4zh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-tlzh4bzgw4zh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n \"name\": \"3547c17b-2baf-4dfa-a1b2-187f3ed372c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T10:04:17.573277188Z\",\n \"endTime\": \"2026-09-26T10:28:26.278499332Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t2kdfqqgc9osqgq67u99r2mbueuta1t still exists (deletion dispatched at 2026-09-26T10:04:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["3de4c85edc65d4851e95835c19adeaf11200c5ea73dde88d529944e306df4303","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103725120211128320","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-26T05:55:28Z","started_at":"2026-09-26T05:55:28Z","completed_at":"2026-09-26T07:48:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T07:57:22.978350604Z","timing_metadata_checked_at":"2026-09-26T07:57:22.978350604Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T07:57:22.983708986Z","region_metadata_checked_at":"2026-09-26T07:57:22.983708986Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"68dc43f8030e405a2b58a00c119849906704aea638d63157c25645bcfa098836","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103725120211128320","occurred_at":"2026-09-26T05:55:28Z","failed_at":"e2e","signature_id":"cb3842f13643a3ede46cd6921339aeccd780d8ffc478c77a73c368f7b7de93b8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tcwpsk\" for cluster \"cluster-candidate-4-21-tcwpsk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000219700\u003e: \n failed to create NodePool np-tcwpsk: failed waiting for nodepool=\"np-tcwpsk\" for cluster \"cluster-candidate-4-21-tcwpsk\" in resourcegroup=\"rg-candidate-4-21-fcmqz6t9cprm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6626f1a6-57db-468a-9a1e-3a38d2373d87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6626f1a6-57db-468a-9a1e-3a38d2373d87\",\n \"name\": \"6626f1a6-57db-468a-9a1e-3a38d2373d87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T06:35:50.80138522Z\",\n \"endTime\": \"2026-09-26T06:54:54.507616611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u4m5g1d1o4y5j2t-np-tcwpsk-474vr-hcjmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-fcmqz6t9cprm-candidate-tcwpsk--managed/providers/Microsoft.Compute/virtualMachines/u4m5g1d1o4y5j2t-np-tcwpsk-474vr-hcjmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u4m5g1d1o4y5j2t-np-tcwpsk-474vr-hcjmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-fcmqz6t9cprm-candidate-tcwpsk--managed/providers/Microsoft.Compute/virtualMachines/u4m5g1d1o4y5j2t-np-tcwpsk-474vr-hcjmx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tcwpsk\" for cluster \"cluster-candidate-4-21-tcwpsk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000219700\u003e: \n failed to create NodePool np-tcwpsk: failed waiting for nodepool=\"np-tcwpsk\" for cluster \"cluster-candidate-4-21-tcwpsk\" in resourcegroup=\"rg-candidate-4-21-fcmqz6t9cprm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6626f1a6-57db-468a-9a1e-3a38d2373d87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6626f1a6-57db-468a-9a1e-3a38d2373d87\",\n \"name\": \"6626f1a6-57db-468a-9a1e-3a38d2373d87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T06:35:50.80138522Z\",\n \"endTime\": \"2026-09-26T06:54:54.507616611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u4m5g1d1o4y5j2t-np-tcwpsk-474vr-hcjmx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-fcmqz6t9cprm-candidate-tcwpsk--managed/providers/Microsoft.Compute/virtualMachines/u4m5g1d1o4y5j2t-np-tcwpsk-474vr-hcjmx to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u4m5g1d1o4y5j2t-np-tcwpsk-474vr-hcjmx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-fcmqz6t9cprm-candidate-tcwpsk--managed/providers/Microsoft.Compute/virtualMachines/u4m5g1d1o4y5j2t-np-tcwpsk-474vr-hcjmx to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a735e6e453be8a1b2e4e8daaa32a755e76ef3b08fcff7c1c0c2f5b582453d14d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103725120211128320","occurred_at":"2026-09-26T05:55:28Z","failed_at":"e2e","signature_id":"306b29e323c6a56937a91e8a9b592e4457743d2ec8d8a321540ac9c29f5c5317","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1aea0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-l5wcbthtdvrj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc4520f0-59dd-4c27-b6ac-3a37c3d68b61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc4520f0-59dd-4c27-b6ac-3a37c3d68b61\",\n \"name\": \"fc4520f0-59dd-4c27-b6ac-3a37c3d68b61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T06:10:39.440155769Z\",\n \"endTime\": \"2026-09-26T06:29:48.524896029Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r2a4d5d1z0i7s2r-np-1-9q5wc-dlv85: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-l5wcbthtdvrj--managed/providers/Microsoft.Compute/virtualMachines/r2a4d5d1z0i7s2r-np-1-9q5wc-dlv85 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r2a4d5d1z0i7s2r-np-1-9q5wc-dlv85: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-l5wcbthtdvrj--managed/providers/Microsoft.Compute/virtualMachines/r2a4d5d1z0i7s2r-np-1-9q5wc-dlv85 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1aea0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-l5wcbthtdvrj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc4520f0-59dd-4c27-b6ac-3a37c3d68b61\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc4520f0-59dd-4c27-b6ac-3a37c3d68b61\",\n \"name\": \"fc4520f0-59dd-4c27-b6ac-3a37c3d68b61\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T06:10:39.440155769Z\",\n \"endTime\": \"2026-09-26T06:29:48.524896029Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r2a4d5d1z0i7s2r-np-1-9q5wc-dlv85: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-l5wcbthtdvrj--managed/providers/Microsoft.Compute/virtualMachines/r2a4d5d1z0i7s2r-np-1-9q5wc-dlv85 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r2a4d5d1z0i7s2r-np-1-9q5wc-dlv85: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-l5wcbthtdvrj--managed/providers/Microsoft.Compute/virtualMachines/r2a4d5d1z0i7s2r-np-1-9q5wc-dlv85 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103690040713416704","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-26T03:36:04Z","started_at":"2026-09-26T03:36:04Z","completed_at":"2026-09-26T05:49:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T06:01:26.299627159Z","timing_metadata_checked_at":"2026-09-26T06:01:26.299627159Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T06:01:26.307945931Z","region_metadata_checked_at":"2026-09-26T06:01:26.307945931Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"dd054f66fc9778938d14e0c3d50f54796cc96bdb6b65af4e75a4d92fdcd2c84f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103690040713416704","occurred_at":"2026-09-26T03:36:04Z","failed_at":"e2e","signature_id":"038e89abf06c9fb71bc46c4424eb7ecab72c651acda302bd05745bbec4db2f13","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2c4h96\" for cluster \"cluster-candidate-4-23-2c4h96\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a68b20\u003e: \n failed to create NodePool np-2c4h96: failed waiting for nodepool=\"np-2c4h96\" for cluster \"cluster-candidate-4-23-2c4h96\" in resourcegroup=\"rg-candidate-4-23-54tf9r44kbdx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/084dd236-2858-409a-b7ec-4a1409b0e6b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/084dd236-2858-409a-b7ec-4a1409b0e6b6\",\n \"name\": \"084dd236-2858-409a-b7ec-4a1409b0e6b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T04:37:04.007379792Z\",\n \"endTime\": \"2026-09-26T04:56:04.017294795Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h4g8t4x4z5g9w3y-np-2c4h96-btvvm-wk24r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-54tf9r44kbdx-candidate-2c4h96--managed/providers/Microsoft.Compute/virtualMachines/h4g8t4x4z5g9w3y-np-2c4h96-btvvm-wk24r to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h4g8t4x4z5g9w3y-np-2c4h96-btvvm-wk24r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-54tf9r44kbdx-candidate-2c4h96--managed/providers/Microsoft.Compute/virtualMachines/h4g8t4x4z5g9w3y-np-2c4h96-btvvm-wk24r to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2c4h96\" for cluster \"cluster-candidate-4-23-2c4h96\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a68b20\u003e: \n failed to create NodePool np-2c4h96: failed waiting for nodepool=\"np-2c4h96\" for cluster \"cluster-candidate-4-23-2c4h96\" in resourcegroup=\"rg-candidate-4-23-54tf9r44kbdx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/084dd236-2858-409a-b7ec-4a1409b0e6b6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/084dd236-2858-409a-b7ec-4a1409b0e6b6\",\n \"name\": \"084dd236-2858-409a-b7ec-4a1409b0e6b6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T04:37:04.007379792Z\",\n \"endTime\": \"2026-09-26T04:56:04.017294795Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h4g8t4x4z5g9w3y-np-2c4h96-btvvm-wk24r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-54tf9r44kbdx-candidate-2c4h96--managed/providers/Microsoft.Compute/virtualMachines/h4g8t4x4z5g9w3y-np-2c4h96-btvvm-wk24r to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h4g8t4x4z5g9w3y-np-2c4h96-btvvm-wk24r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-54tf9r44kbdx-candidate-2c4h96--managed/providers/Microsoft.Compute/virtualMachines/h4g8t4x4z5g9w3y-np-2c4h96-btvvm-wk24r to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2103665929907867648","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-26T02:00:16Z","started_at":"2026-09-26T02:00:16Z","completed_at":"2026-09-26T03:54:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-26T04:05:50.432339364Z","timing_metadata_checked_at":"2026-09-26T04:05:50.432339364Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-26T04:05:50.435264854Z","region_metadata_checked_at":"2026-09-26T04:05:50.435264854Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"65d06cc14572ead7ad07c589d78ff4b8e9b071cba4663a1810bd953a902e3ce9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2103665929907867648","occurred_at":"2026-09-26T02:00:16Z","failed_at":"e2e","signature_id":"f981c1008c821f13aae9959c202473fe9605a11d0fdc588a129f3cedf979334e","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b56dc0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-c2cwvz\" in resourcegroup=\"rg-np-noedge-c2cwvz-qj8wgrzd8jpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n \"name\": \"4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T03:07:12.674137276Z\",\n \"endTime\": \"2026-09-26T03:26:19.484267097Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b56dc0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-c2cwvz\" in resourcegroup=\"rg-np-noedge-c2cwvz-qj8wgrzd8jpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n \"name\": \"4d8efa06-245b-47cb-9abb-ec98064be1ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T03:07:12.674137276Z\",\n \"endTime\": \"2026-09-26T03:26:19.484267097Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-c2cwvz-qj8wgrzd8jpf-np-ne-c2cwvz--managed/providers/Microsoft.Compute/virtualMachines/s7l7b3o9i1a8y0f-npnoedge-4-20-t278w-8cm8n to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"96800d7c7de38f007fb6c1e0881939b8ef8d0c398e84b2e8ff15cc298d9bdcd5","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f074f48d18cafea639c635df211a4ae9afd9fdd2a06a93f552fe79e5634f3b58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2103665929907867648","occurred_at":"2026-09-26T02:00:16Z","failed_at":"e2e","signature_id":"61e1a0c5120956316d98b8f902f363d01c50acf5c533fe1e2472e0d4bbca8113","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1ba60\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-89w5k67x22pw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d1d982d-3b36-4d74-abe3-4a35c064fbed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d1d982d-3b36-4d74-abe3-4a35c064fbed\",\n \"name\": \"6d1d982d-3b36-4d74-abe3-4a35c064fbed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T03:09:42.046607083Z\",\n \"endTime\": \"2026-09-26T03:28:46.487681317Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-jb75r-zxj2z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-89w5k67x22pw--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-jb75r-zxj2z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-jb75r-zxj2z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-89w5k67x22pw--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-jb75r-zxj2z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1ba60\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-89w5k67x22pw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d1d982d-3b36-4d74-abe3-4a35c064fbed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d1d982d-3b36-4d74-abe3-4a35c064fbed\",\n \"name\": \"6d1d982d-3b36-4d74-abe3-4a35c064fbed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-26T03:09:42.046607083Z\",\n \"endTime\": \"2026-09-26T03:28:46.487681317Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-jb75r-zxj2z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-89w5k67x22pw--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-jb75r-zxj2z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-jb75r-zxj2z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-89w5k67x22pw--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-jb75r-zxj2z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["96800d7c7de38f007fb6c1e0881939b8ef8d0c398e84b2e8ff15cc298d9bdcd5","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}