{"meta":{"date":"2026-08-11","timezone":"UTC","generated_at":"2026-08-26T18:26:23Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-11T23:00:11Z"},"failed_at":["e2e"],"failed_test_count":32,"occurrences":[{"row_id":"040d28b1c59938d6a943e1d95369b28f251b976d1afa544f0a7f9731ee58a3eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"9f71ed71ec351f3e09b671a12521f6921104fae6b073d5a569b54b2f2dc038dd","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-v8v4bw at version 4.22.0-0.nightly-multi-2026-08-11-120309\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f268e0\u003e: \n    failed to create HCP cluster np-skip-minor-v8v4bw: failed waiting for cluster=\"np-skip-minor-v8v4bw\" in resourcegroup=\"rg-np-skip-minor-v8v4bw-m9qsqfcg26w6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d8c33446-8f67-46eb-8d4f-70a6f9dd3471\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d8c33446-8f67-46eb-8d4f-70a6f9dd3471\",\n      \"name\": \"d8c33446-8f67-46eb-8d4f-70a6f9dd3471\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:50:01.086809244Z\",\n      \"endTime\": \"2026-08-12T00:09:06.701618581Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-v8v4bw at version 4.22.0-0.nightly-multi-2026-08-11-120309\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f268e0\u003e: \n    failed to create HCP cluster np-skip-minor-v8v4bw: failed waiting for cluster=\"np-skip-minor-v8v4bw\" in resourcegroup=\"rg-np-skip-minor-v8v4bw-m9qsqfcg26w6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d8c33446-8f67-46eb-8d4f-70a6f9dd3471\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d8c33446-8f67-46eb-8d4f-70a6f9dd3471\",\n      \"name\": \"d8c33446-8f67-46eb-8d4f-70a6f9dd3471\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:50:01.086809244Z\",\n      \"endTime\": \"2026-08-12T00:09:06.701618581Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"189a61d70465a6f21c32847cacfcca929b4cd22ee2317c4c6a0a27cdfec1991d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"e2581a1a2524d2a89277b04dcce67f35f092e92c6b733e648a9c645aab9a694b","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ec72a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-872q6c5pdbsv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f84fe915-64a6-40a0-a041-589a090184bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f84fe915-64a6-40a0-a041-589a090184bf\",\n      \"name\": \"f84fe915-64a6-40a0-a041-589a090184bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:36:01.387144352Z\",\n      \"endTime\": \"2026-08-11T23:55:06.812193946Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ec72a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-872q6c5pdbsv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f84fe915-64a6-40a0-a041-589a090184bf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f84fe915-64a6-40a0-a041-589a090184bf\",\n      \"name\": \"f84fe915-64a6-40a0-a041-589a090184bf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:36:01.387144352Z\",\n      \"endTime\": \"2026-08-11T23:55:06.812193946Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2231cc21085dedff661cf97cbb86333cc8d456aa9c164422ca5d5154576c3d29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"7cc0f7687301edc8d9f72e3c241cd0fe76be25e35ab28ef85fc016aca0c09eb0","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005a0360\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-2295pqr4vnqk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df22c022-84b2-42e9-b743-273877a0d7b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df22c022-84b2-42e9-b743-273877a0d7b0\",\n      \"name\": \"df22c022-84b2-42e9-b743-273877a0d7b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:36:06.917019748Z\",\n      \"endTime\": \"2026-08-11T23:55:06.835167499Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005a0360\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-2295pqr4vnqk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df22c022-84b2-42e9-b743-273877a0d7b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df22c022-84b2-42e9-b743-273877a0d7b0\",\n      \"name\": \"df22c022-84b2-42e9-b743-273877a0d7b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:36:06.917019748Z\",\n      \"endTime\": \"2026-08-11T23:55:06.835167499Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2a7b97d83146641137a13285053f6843e8c24248c1e546a3846f8d7dcc7ac551","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"90d885c06d2a58ee01a59b61bd0c516a62d14cee5fde5b964d99f4be99f0e01c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-89szk8\" with version 4.20.32 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c058c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-89szk8: failed waiting for cluster=\"cluster-zstream-4-20-89szk8\" in resourcegroup=\"rg-zstream-upgrade-4-20-vs42d9hw65cs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28c61d74-e56f-42f2-9428-b4ee03170bfb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28c61d74-e56f-42f2-9428-b4ee03170bfb\",\n      \"name\": \"28c61d74-e56f-42f2-9428-b4ee03170bfb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:28:06.494455039Z\",\n      \"endTime\": \"2026-08-11T23:47:06.337628951Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-89szk8\" with version 4.20.32 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c058c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-89szk8: failed waiting for cluster=\"cluster-zstream-4-20-89szk8\" in resourcegroup=\"rg-zstream-upgrade-4-20-vs42d9hw65cs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28c61d74-e56f-42f2-9428-b4ee03170bfb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/28c61d74-e56f-42f2-9428-b4ee03170bfb\",\n      \"name\": \"28c61d74-e56f-42f2-9428-b4ee03170bfb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:28:06.494455039Z\",\n      \"endTime\": \"2026-08-11T23:47:06.337628951Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2eeb884664d3648bb1f6f4623e185cccb7cc1dbc4a1d188f38dc3965ad4e0966","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"645f29ee8a8529d864d5611c1a886f10eb6db0b4b084b3afab46380bfd91dbb0","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e16880\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8c26dnrlrzdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b99e8701-f6d9-49e5-83c7-013d841d976b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b99e8701-f6d9-49e5-83c7-013d841d976b\",\n      \"name\": \"b99e8701-f6d9-49e5-83c7-013d841d976b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:50:25.381567708Z\",\n      \"endTime\": \"2026-08-12T00:09:26.720382941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e16880\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-8c26dnrlrzdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b99e8701-f6d9-49e5-83c7-013d841d976b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b99e8701-f6d9-49e5-83c7-013d841d976b\",\n      \"name\": \"b99e8701-f6d9-49e5-83c7-013d841d976b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:50:25.381567708Z\",\n      \"endTime\": \"2026-08-12T00:09:26.720382941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3c25ebb1415aa51ff5ea461bee5e734360f499106890ba18115c431f33cf2208","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"e51c6d9d8ed0abf2f24fdbff63f137b9a70cc148c6d6f76c1884689eebdd7590","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a864e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-c5xnftjhfp8n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72c120d3-7fe5-4b56-bf50-717ac085c652\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72c120d3-7fe5-4b56-bf50-717ac085c652\",\n      \"name\": \"72c120d3-7fe5-4b56-bf50-717ac085c652\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:51:57.787726414Z\",\n      \"endTime\": \"2026-08-12T00:11:06.769461941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a864e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-c5xnftjhfp8n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72c120d3-7fe5-4b56-bf50-717ac085c652\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/72c120d3-7fe5-4b56-bf50-717ac085c652\",\n      \"name\": \"72c120d3-7fe5-4b56-bf50-717ac085c652\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:51:57.787726414Z\",\n      \"endTime\": \"2026-08-12T00:11:06.769461941Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3ed69a43c3006815ffe8c92f9a079c72c363d1b11a6c1069545d2e9e5452200a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"7aa03aa095785242e3917e289f5a8c403efc81ed0ce26dde9fdd46a0aa058587","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-21-h8hmjt9f5hbv/cluster-nightly-4-21-8d2hpg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1ef00\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-8d2hpg: failed waiting for cluster=\"cluster-nightly-4-21-8d2hpg\" in resourcegroup=\"rg-nightly-4-21-h8hmjt9f5hbv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1f68e48e-59be-457f-827d-2ead4fee0374\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1f68e48e-59be-457f-827d-2ead4fee0374\",\n      \"name\": \"1f68e48e-59be-457f-827d-2ead4fee0374\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:26:46.06168874Z\",\n      \"endTime\": \"2026-08-11T23:45:46.38773987Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-21-h8hmjt9f5hbv/cluster-nightly-4-21-8d2hpg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1ef00\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-8d2hpg: failed waiting for cluster=\"cluster-nightly-4-21-8d2hpg\" in resourcegroup=\"rg-nightly-4-21-h8hmjt9f5hbv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1f68e48e-59be-457f-827d-2ead4fee0374\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1f68e48e-59be-457f-827d-2ead4fee0374\",\n      \"name\": \"1f68e48e-59be-457f-827d-2ead4fee0374\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:26:46.06168874Z\",\n      \"endTime\": \"2026-08-11T23:45:46.38773987Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"461988b81a03294ff5e2d377ad37b376beb6cc80590d92952e20bb0853c2bc60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"8f3c94881f6801e2505c0c36c4f711450bdeb3fbb4e496f461c291ba62f190af","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a061e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gnfmr9nz8xkt\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ba17e68-ad24-4cc5-83e9-15647fb0ca73\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ba17e68-ad24-4cc5-83e9-15647fb0ca73\",\n      \"name\": \"8ba17e68-ad24-4cc5-83e9-15647fb0ca73\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:24:43.846421644Z\",\n      \"endTime\": \"2026-08-11T23:48:46.480512763Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lom1h1084lkfrkvqud03tkqmhb1pn still exists (deletion dispatched at 2026-08-11T23:24:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a061e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gnfmr9nz8xkt\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ba17e68-ad24-4cc5-83e9-15647fb0ca73\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ba17e68-ad24-4cc5-83e9-15647fb0ca73\",\n      \"name\": \"8ba17e68-ad24-4cc5-83e9-15647fb0ca73\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:24:43.846421644Z\",\n      \"endTime\": \"2026-08-11T23:48:46.480512763Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lom1h1084lkfrkvqud03tkqmhb1pn still exists (deletion dispatched at 2026-08-11T23:24:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"474cdcc6c4387867d47cf8b7b73c301629c1dd0c2bd0ecb81f336c5371414c27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"c83ed80c08140c9ce56f7edae806c3868c1411d4309a288d5ab8b1fdd766478f","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011e1278\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-6q964t878t5c\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8268305c-a869-47cb-9f39-52837fcdc35e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8268305c-a869-47cb-9f39-52837fcdc35e\",\n      \"name\": \"8268305c-a869-47cb-9f39-52837fcdc35e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:29:50.798521613Z\",\n      \"endTime\": \"2026-08-11T23:53:56.910068365Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lo8lv4gtesak1jnmu9b9k11fo9402 still exists (deletion dispatched at 2026-08-11T23:29:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011e1278\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-6q964t878t5c\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8268305c-a869-47cb-9f39-52837fcdc35e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8268305c-a869-47cb-9f39-52837fcdc35e\",\n      \"name\": \"8268305c-a869-47cb-9f39-52837fcdc35e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:29:50.798521613Z\",\n      \"endTime\": \"2026-08-11T23:53:56.910068365Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lo8lv4gtesak1jnmu9b9k11fo9402 still exists (deletion dispatched at 2026-08-11T23:29:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b936783c2d0b2a9d4ffabaeda14053cc9fef881b92be87e0e230002db2affbaf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4d9f1266b15eb0efccff54dcfa6916deddad0b60c9dc38f4be4f069475918a68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"9cabd00401d613a32fb974fa81fb9fd3824ab5457eae1e8b6dbe896d65b4a925","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:741]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e20c00\u003e: \n    failed to create HCP cluster np-dg-minor-455shf: failed waiting for cluster=\"np-dg-minor-455shf\" in resourcegroup=\"rg-np-dg-minor-455shf-5bhhf9w6t5jf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2565934-591c-460e-9dd0-16be2a7b1f38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2565934-591c-460e-9dd0-16be2a7b1f38\",\n      \"name\": \"e2565934-591c-460e-9dd0-16be2a7b1f38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:49:53.638521938Z\",\n      \"endTime\": \"2026-08-12T00:08:56.6357084Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:741]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e20c00\u003e: \n    failed to create HCP cluster np-dg-minor-455shf: failed waiting for cluster=\"np-dg-minor-455shf\" in resourcegroup=\"rg-np-dg-minor-455shf-5bhhf9w6t5jf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2565934-591c-460e-9dd0-16be2a7b1f38\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2565934-591c-460e-9dd0-16be2a7b1f38\",\n      \"name\": \"e2565934-591c-460e-9dd0-16be2a7b1f38\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:49:53.638521938Z\",\n      \"endTime\": \"2026-08-12T00:08:56.6357084Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"505970a830f2e36d2047b9de0b85cd5332705a21eefa7ad6d7dcf388a8c12b0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"2bd0cbba9168a292396488ffbcce3df044aec8c5d45aafe042c89ed004c65c25","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-0-hk694lqlw2bk/cluster-nightly-5-0-49kj2j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00097afe0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-49kj2j: failed waiting for cluster=\"cluster-nightly-5-0-49kj2j\" in resourcegroup=\"rg-nightly-5-0-hk694lqlw2bk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/19e3a33f-1e44-4c92-87b9-19e748c70816\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/19e3a33f-1e44-4c92-87b9-19e748c70816\",\n      \"name\": \"19e3a33f-1e44-4c92-87b9-19e748c70816\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:26:58.710386207Z\",\n      \"endTime\": \"2026-08-11T23:46:06.408061301Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-0-hk694lqlw2bk/cluster-nightly-5-0-49kj2j should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00097afe0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-49kj2j: failed waiting for cluster=\"cluster-nightly-5-0-49kj2j\" in resourcegroup=\"rg-nightly-5-0-hk694lqlw2bk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/19e3a33f-1e44-4c92-87b9-19e748c70816\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/19e3a33f-1e44-4c92-87b9-19e748c70816\",\n      \"name\": \"19e3a33f-1e44-4c92-87b9-19e748c70816\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:26:58.710386207Z\",\n      \"endTime\": \"2026-08-11T23:46:06.408061301Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"55dd2c33199a3c999a65199a5e489555f4884cae03033339fa0d28d611c71d82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"6feb0cf106c4603c56edd3f0d3e001b2942a0fa5a286213e006f436823fbec51","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e626a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-rsnbnclf66qj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/39698eca-af1a-43ec-8f2f-9a71b7e5e296\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/39698eca-af1a-43ec-8f2f-9a71b7e5e296\",\n      \"name\": \"39698eca-af1a-43ec-8f2f-9a71b7e5e296\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:50:06.553117602Z\",\n      \"endTime\": \"2026-08-12T00:09:06.716633972Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e626a0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-rsnbnclf66qj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/39698eca-af1a-43ec-8f2f-9a71b7e5e296\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/39698eca-af1a-43ec-8f2f-9a71b7e5e296\",\n      \"name\": \"39698eca-af1a-43ec-8f2f-9a71b7e5e296\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:50:06.553117602Z\",\n      \"endTime\": \"2026-08-12T00:09:06.716633972Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5e84d8afcd0a4ad91e255b2a5d7c08c405b834f7e3ef9119f8788292e394255d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"5e37f71209ab3fdff0f528d346d21e0222b6b5e3461c7dce5c489a742fe18693","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dcb420\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-2v545vp45wtx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c52419f-fbfe-41f6-a404-3f49c8b7edaa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c52419f-fbfe-41f6-a404-3f49c8b7edaa\",\n      \"name\": \"5c52419f-fbfe-41f6-a404-3f49c8b7edaa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:34:00.550612667Z\",\n      \"endTime\": \"2026-08-11T23:53:06.747237705Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dcb420\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-2v545vp45wtx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c52419f-fbfe-41f6-a404-3f49c8b7edaa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c52419f-fbfe-41f6-a404-3f49c8b7edaa\",\n      \"name\": \"5c52419f-fbfe-41f6-a404-3f49c8b7edaa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:34:00.550612667Z\",\n      \"endTime\": \"2026-08-11T23:53:06.747237705Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"62e2b9040f7c11bb59f0fb8fef8e7ce0b09d666b488f15634c14a2f71cc7714e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"b85c5a4baf62aaa5c6dc06ce1161c81145532c628124581ea6987ef89f82d4ca","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00082fa20\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-jr9ctpwlfwjr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7091a9a5-13c6-430f-93d0-17113506a6ff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7091a9a5-13c6-430f-93d0-17113506a6ff\",\n      \"name\": \"7091a9a5-13c6-430f-93d0-17113506a6ff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:34:01.431488993Z\",\n      \"endTime\": \"2026-08-11T23:53:06.97074243Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00082fa20\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-jr9ctpwlfwjr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7091a9a5-13c6-430f-93d0-17113506a6ff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7091a9a5-13c6-430f-93d0-17113506a6ff\",\n      \"name\": \"7091a9a5-13c6-430f-93d0-17113506a6ff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:34:01.431488993Z\",\n      \"endTime\": \"2026-08-11T23:53:06.97074243Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"64b2c2482ba9cecd5e8647684f4ccf4bd842b5d1e7fb5e12d25900ba5719e0d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"3457f6d8e750065d86a843c1b16eadbf0565e0a1839247ff6d297a7cc77459f4","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-19-r26nlz\" with version 4.19.40 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b26e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-r26nlz: failed waiting for cluster=\"cluster-zstream-4-19-r26nlz\" in resourcegroup=\"rg-zstream-upgrade-4-19-mrqbw599ztjb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3d85f866-fcb3-4659-93a4-75a2ad848708\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3d85f866-fcb3-4659-93a4-75a2ad848708\",\n      \"name\": \"3d85f866-fcb3-4659-93a4-75a2ad848708\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:28:14.107965535Z\",\n      \"endTime\": \"2026-08-11T23:47:16.637853926Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-19-r26nlz\" with version 4.19.40 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004b26e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-r26nlz: failed waiting for cluster=\"cluster-zstream-4-19-r26nlz\" in resourcegroup=\"rg-zstream-upgrade-4-19-mrqbw599ztjb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3d85f866-fcb3-4659-93a4-75a2ad848708\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3d85f866-fcb3-4659-93a4-75a2ad848708\",\n      \"name\": \"3d85f866-fcb3-4659-93a4-75a2ad848708\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:28:14.107965535Z\",\n      \"endTime\": \"2026-08-11T23:47:16.637853926Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6fe237025d2ca85b8163dc1017fc32db74e35c2784c8d370c4b6626c487bdaae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"f3b847bf48278e73317c6410942d10c361131f00348076401ea1f1d7110263b8","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8d380\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-9x8xz57k25cx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1424c2b5-7e21-4415-9741-9d6ad91e0b8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1424c2b5-7e21-4415-9741-9d6ad91e0b8d\",\n      \"name\": \"1424c2b5-7e21-4415-9741-9d6ad91e0b8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:32:53.502571214Z\",\n      \"endTime\": \"2026-08-11T23:51:56.487688874Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f8d380\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-9x8xz57k25cx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1424c2b5-7e21-4415-9741-9d6ad91e0b8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1424c2b5-7e21-4415-9741-9d6ad91e0b8d\",\n      \"name\": \"1424c2b5-7e21-4415-9741-9d6ad91e0b8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:32:53.502571214Z\",\n      \"endTime\": \"2026-08-11T23:51:56.487688874Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"81110ac19bfa54d9aa12f9192646b6e1853164325205bde0a491705fc2ebb95a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"b13f46bc78bb9e93f3d4ca7fe7213641ab3fb59274e5ea73d4e8dfc093d6893a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-22-2rvg98ms52gw/cluster-nightly-4-22-9ttt7x should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014141c0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-9ttt7x: failed waiting for cluster=\"cluster-nightly-4-22-9ttt7x\" in resourcegroup=\"rg-nightly-4-22-2rvg98ms52gw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14b29228-55ad-41bc-9514-5c375d336b03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14b29228-55ad-41bc-9514-5c375d336b03\",\n      \"name\": \"14b29228-55ad-41bc-9514-5c375d336b03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:26:56.637572124Z\",\n      \"endTime\": \"2026-08-11T23:45:56.116898146Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-22-2rvg98ms52gw/cluster-nightly-4-22-9ttt7x should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014141c0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-9ttt7x: failed waiting for cluster=\"cluster-nightly-4-22-9ttt7x\" in resourcegroup=\"rg-nightly-4-22-2rvg98ms52gw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14b29228-55ad-41bc-9514-5c375d336b03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/14b29228-55ad-41bc-9514-5c375d336b03\",\n      \"name\": \"14b29228-55ad-41bc-9514-5c375d336b03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:26:56.637572124Z\",\n      \"endTime\": \"2026-08-11T23:45:56.116898146Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"88303512f284ce908fdaefe957182a09f798cfd3277a5616be6edcbe9cc1d515","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"fd8cc01860168bdf9b04e4731fb16167c71af34105f5ac37c300c6a69e546753","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005a85c0\u003e: \n    failed to create HCP cluster ea-listtndr27: failed waiting for cluster=\"ea-listtndr27\" in resourcegroup=\"ea-list-h87vt96xv6bf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5416ae06-e78d-49c0-a38d-0a968ebbca47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5416ae06-e78d-49c0-a38d-0a968ebbca47\",\n      \"name\": \"5416ae06-e78d-49c0-a38d-0a968ebbca47\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:33:50.736356948Z\",\n      \"endTime\": \"2026-08-11T23:52:56.551902262Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005a85c0\u003e: \n    failed to create HCP cluster ea-listtndr27: failed waiting for cluster=\"ea-listtndr27\" in resourcegroup=\"ea-list-h87vt96xv6bf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5416ae06-e78d-49c0-a38d-0a968ebbca47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5416ae06-e78d-49c0-a38d-0a968ebbca47\",\n      \"name\": \"5416ae06-e78d-49c0-a38d-0a968ebbca47\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:33:50.736356948Z\",\n      \"endTime\": \"2026-08-11T23:52:56.551902262Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"98121efbdb107cf5a3cbffe171bb485d6511b5b7bffa5b0341b93b1b3cebdf07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"30d78abd1e8effc50aa9aaaaeed7dcab0a85148c0d328b2400522ae91cbcc13e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-kzxlwf\" with version 4.21.27 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc3360\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-kzxlwf: failed waiting for cluster=\"cluster-zstream-4-21-kzxlwf\" in resourcegroup=\"rg-zstream-upgrade-4-21-ctkvqphq82sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd26e343-cdb7-421e-a58e-8adc8543ee23\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd26e343-cdb7-421e-a58e-8adc8543ee23\",\n      \"name\": \"bd26e343-cdb7-421e-a58e-8adc8543ee23\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:30:51.25570724Z\",\n      \"endTime\": \"2026-08-11T23:49:56.4117678Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-kzxlwf\" with version 4.21.27 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc3360\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-kzxlwf: failed waiting for cluster=\"cluster-zstream-4-21-kzxlwf\" in resourcegroup=\"rg-zstream-upgrade-4-21-ctkvqphq82sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd26e343-cdb7-421e-a58e-8adc8543ee23\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd26e343-cdb7-421e-a58e-8adc8543ee23\",\n      \"name\": \"bd26e343-cdb7-421e-a58e-8adc8543ee23\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:30:51.25570724Z\",\n      \"endTime\": \"2026-08-11T23:49:56.4117678Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9b6f1e9eb3a87cbfe1efe9c982ddb2a337f9d249d7e994194082a96ee4eec2aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"21c2aa7cd4ff06247870bf7447c7b75ab5ea241b361e5971af57f6b732e7bc27","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002c381b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-hx4vsq\" in resourcegroup=\"rg-nightly-4-20-mlf9649gf6nw\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b353c4f6-44af-4a58-95ee-686b06d67560\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b353c4f6-44af-4a58-95ee-686b06d67560\",\n      \"name\": \"b353c4f6-44af-4a58-95ee-686b06d67560\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:24:36.608917138Z\",\n      \"endTime\": \"2026-08-11T23:48:36.565035314Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lnvigvp182t3h8fe2d8ebnbeppjfr still exists (deletion dispatched at 2026-08-11T23:24:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc002c381b0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-20-hx4vsq\" in resourcegroup=\"rg-nightly-4-20-mlf9649gf6nw\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b353c4f6-44af-4a58-95ee-686b06d67560\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b353c4f6-44af-4a58-95ee-686b06d67560\",\n      \"name\": \"b353c4f6-44af-4a58-95ee-686b06d67560\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:24:36.608917138Z\",\n      \"endTime\": \"2026-08-11T23:48:36.565035314Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lnvigvp182t3h8fe2d8ebnbeppjfr still exists (deletion dispatched at 2026-08-11T23:24:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a18bed693c07b86845f16b74d166fc504362144e39b76fd241321cfb516f880c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"71464b00be31c8c631f96da2e23a189636b56b30c6c800668733df00c3b55b22","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0000126a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-tf5ktrtjbh5z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1edc4471-f6a7-4bf4-9a97-0e58c926d25f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1edc4471-f6a7-4bf4-9a97-0e58c926d25f\",\n      \"name\": \"1edc4471-f6a7-4bf4-9a97-0e58c926d25f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:35:18.055944742Z\",\n      \"endTime\": \"2026-08-11T23:59:27.915516747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lpbkv0a8sakjqk2n5gd31klhfru7u still exists (deletion dispatched at 2026-08-11T23:35:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0000126a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-tf5ktrtjbh5z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1edc4471-f6a7-4bf4-9a97-0e58c926d25f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1edc4471-f6a7-4bf4-9a97-0e58c926d25f\",\n      \"name\": \"1edc4471-f6a7-4bf4-9a97-0e58c926d25f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:35:18.055944742Z\",\n      \"endTime\": \"2026-08-11T23:59:27.915516747Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lpbkv0a8sakjqk2n5gd31klhfru7u still exists (deletion dispatched at 2026-08-11T23:35:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b936783c2d0b2a9d4ffabaeda14053cc9fef881b92be87e0e230002db2affbaf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a3fbc45bff1b6da1766047d6b2193f827ad6f499f897ca4a10e368ee9e2b0f2b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"4549996248c9ca47388937606e862baa031b2cc38e59fcf4ff818e2c5727388f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-f88jxn\" with version 4.22.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00092d3a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-f88jxn: failed waiting for cluster=\"cluster-zstream-4-22-f88jxn\" in resourcegroup=\"rg-zstream-upgrade-4-22-p6f9z7g7lkxq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/baac4f53-12cd-4b8b-9236-ffa522b8ad35\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/baac4f53-12cd-4b8b-9236-ffa522b8ad35\",\n      \"name\": \"baac4f53-12cd-4b8b-9236-ffa522b8ad35\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:30:51.638118376Z\",\n      \"endTime\": \"2026-08-11T23:49:56.400834325Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-f88jxn\" with version 4.22.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00092d3a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-f88jxn: failed waiting for cluster=\"cluster-zstream-4-22-f88jxn\" in resourcegroup=\"rg-zstream-upgrade-4-22-p6f9z7g7lkxq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/baac4f53-12cd-4b8b-9236-ffa522b8ad35\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/baac4f53-12cd-4b8b-9236-ffa522b8ad35\",\n      \"name\": \"baac4f53-12cd-4b8b-9236-ffa522b8ad35\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:30:51.638118376Z\",\n      \"endTime\": \"2026-08-11T23:49:56.400834325Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a597bec0f202750d1bfd0ba2b426ddf45904ee438a30c4a8f5a045b2979be019","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"11e97a94ab644e270f869f43f54d365e4f3e8a897ceed345a3bcdc60888eb9cf","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008929e0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k7fpnkzrkndh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3a5bda5-d7d0-485b-a01a-88b283314b01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3a5bda5-d7d0-485b-a01a-88b283314b01\",\n      \"name\": \"a3a5bda5-d7d0-485b-a01a-88b283314b01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:48:31.414599631Z\",\n      \"endTime\": \"2026-08-12T00:07:36.648446945Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008929e0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k7fpnkzrkndh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3a5bda5-d7d0-485b-a01a-88b283314b01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a3a5bda5-d7d0-485b-a01a-88b283314b01\",\n      \"name\": \"a3a5bda5-d7d0-485b-a01a-88b283314b01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:48:31.414599631Z\",\n      \"endTime\": \"2026-08-12T00:07:36.648446945Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b79bc9b0842165e98f032c6ef7e546ead0c17eb48af7f1a423d8bfbae13cb02f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"ead946b963002456de20e0fca86247e972c98ac0df99e0185ce4fb8a39770e39","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-ftqqnccj9lnc/cluster-nightly-5-1-fnn2fp should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013eb290\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-fnn2fp, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-fnn2fp in resource group rg-nightly-5-1-ftqqnccj9lnc, error: failed waiting for cluster=\"cluster-nightly-5-1-fnn2fp\" in resourcegroup=\"rg-nightly-5-1-ftqqnccj9lnc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-fnn2fp in resource group rg-nightly-5-1-ftqqnccj9lnc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-ftqqnccj9lnc/cluster-nightly-5-1-fnn2fp should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013eb290\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-fnn2fp, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-fnn2fp in resource group rg-nightly-5-1-ftqqnccj9lnc, error: failed waiting for cluster=\"cluster-nightly-5-1-fnn2fp\" in resourcegroup=\"rg-nightly-5-1-ftqqnccj9lnc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-fnn2fp in resource group rg-nightly-5-1-ftqqnccj9lnc, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-fnn2fp, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-fnn2fp in resource group rg-nightly-5-1-ftqqnccj9lnc, error: failed waiting for cluster=\"cluster-nightly-5-1-fnn2fp\" in resourcegroup=\"rg-nightly-5-1-ftqqnccj9lnc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-fnn2fp in resource group rg-nightly-5-1-ftqqnccj9lnc, error: context deadline exceeded"}},{"row_id":"c41e6658b3d8f877cf9147940b2cc397040e8ac59122474bdf4090d1e99ae972","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"de6dc19455e6a3ba660c61d5438db7f4b302f72bbdd0a925d1c1c8a200a58df4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-23-jk26knd6fgzn/cluster-nightly-4-23-ksd282 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007904e0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-ksd282: failed waiting for cluster=\"cluster-nightly-4-23-ksd282\" in resourcegroup=\"rg-nightly-4-23-jk26knd6fgzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/50d1d50b-abee-4910-b44e-270b7a53abbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/50d1d50b-abee-4910-b44e-270b7a53abbf\",\n      \"name\": \"50d1d50b-abee-4910-b44e-270b7a53abbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:26:59.323149117Z\",\n      \"endTime\": \"2026-08-11T23:46:06.342978979Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-23-jk26knd6fgzn/cluster-nightly-4-23-ksd282 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007904e0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-ksd282: failed waiting for cluster=\"cluster-nightly-4-23-ksd282\" in resourcegroup=\"rg-nightly-4-23-jk26knd6fgzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/50d1d50b-abee-4910-b44e-270b7a53abbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/50d1d50b-abee-4910-b44e-270b7a53abbf\",\n      \"name\": \"50d1d50b-abee-4910-b44e-270b7a53abbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:26:59.323149117Z\",\n      \"endTime\": \"2026-08-11T23:46:06.342978979Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c6cd8d1f51b410b8e0a5e2f4797dc80618781ae4d68c5de268dff05619bf0b4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"b970e2a0f2dfb35f2fa2dc1303299631a8a76c1eb7b8fb494b6626fb73d9032d","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001cb8270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-txpstthkpqf5\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6988e3d6-531a-403f-b429-debe354d0017\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6988e3d6-531a-403f-b429-debe354d0017\",\n      \"name\": \"6988e3d6-531a-403f-b429-debe354d0017\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:25:10.685424571Z\",\n      \"endTime\": \"2026-08-11T23:49:16.632334065Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lnu714p9rp61esar0vstdaknq7u6m still exists (deletion dispatched at 2026-08-11T23:25:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001cb8270\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-txpstthkpqf5\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6988e3d6-531a-403f-b429-debe354d0017\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6988e3d6-531a-403f-b429-debe354d0017\",\n      \"name\": \"6988e3d6-531a-403f-b429-debe354d0017\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:25:10.685424571Z\",\n      \"endTime\": \"2026-08-11T23:49:16.632334065Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lnu714p9rp61esar0vstdaknq7u6m still exists (deletion dispatched at 2026-08-11T23:25:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c907caa42e4c1f4627c515154947428480e506ffb86f82e72de27f1aa9723d2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"23a180d119748951cb3b729d34f19753d652f0d82f2fb26110f52bec3c9f1539","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001062920\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-zhfqgx7mngp7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/39923cf0-6127-4281-9c3c-756021b4d91a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/39923cf0-6127-4281-9c3c-756021b4d91a\",\n      \"name\": \"39923cf0-6127-4281-9c3c-756021b4d91a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:35:56.77196499Z\",\n      \"endTime\": \"2026-08-11T23:54:56.652443649Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001062920\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-zhfqgx7mngp7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/39923cf0-6127-4281-9c3c-756021b4d91a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/39923cf0-6127-4281-9c3c-756021b4d91a\",\n      \"name\": \"39923cf0-6127-4281-9c3c-756021b4d91a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:35:56.77196499Z\",\n      \"endTime\": \"2026-08-11T23:54:56.652443649Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cc5fbae752fb5af2b734df00d3f7d391eb5b1b3a2b3321f24c9b173d40b55c99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"12abdebb2d6090686fa7e945e31763347d16acf394682545d22dcbe54bb48a3c","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002c18270\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-bmlcnvz9ckpx, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bmlcnvz9ckpx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-bmlcnvz9ckpx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002c18270\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-bmlcnvz9ckpx, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-bmlcnvz9ckpx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-bmlcnvz9ckpx, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-fnn2fp, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-fnn2fp in resource group rg-nightly-5-1-ftqqnccj9lnc, error: failed waiting for cluster=\"cluster-nightly-5-1-fnn2fp\" in resourcegroup=\"rg-nightly-5-1-ftqqnccj9lnc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-fnn2fp in resource group rg-nightly-5-1-ftqqnccj9lnc, error: context deadline exceeded"}},{"row_id":"e3e5f1379383d746c52c0469f31f05d2b12b4a2859d55f47ac2cc40752b50787","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"8bb28762457a3566cfe5883d6969e432d3c5326048d8a72eec27e79b83601a3d","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:146]: failed to delete customer resource group \"cx-rg-cluster-k6m78lg6wgqs\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc003849260\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-k6m78lg6wgqs, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:146]: failed to delete customer resource group \"cx-rg-cluster-k6m78lg6wgqs\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc003849260\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-k6m78lg6wgqs, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6ec87c7d972149dae5ab13de0394af8c0dc25683fd32e5ecca933f42f5cfc45","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-k6m78lg6wgqs, error: context deadline exceeded"}},{"row_id":"e6d237833b914c22e5de05bfa502d41d2626c9351a06ce1d2fc3f7db96651e26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"c5846a675ef416c1c52808f531daa7047ba92253868ddf47fc88fe3c1f7f8894","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001370198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-djvj7mkbzj5m\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a598452-8f61-4a4b-80ae-f7e88c1dabdb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a598452-8f61-4a4b-80ae-f7e88c1dabdb\",\n      \"name\": \"7a598452-8f61-4a4b-80ae-f7e88c1dabdb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:26:45.856883319Z\",\n      \"endTime\": \"2026-08-11T23:50:46.49113577Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lom43tvt8erhq92rpfrdqo0hb0ar9 still exists (deletion dispatched at 2026-08-11T23:26:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001370198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-djvj7mkbzj5m\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a598452-8f61-4a4b-80ae-f7e88c1dabdb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7a598452-8f61-4a4b-80ae-f7e88c1dabdb\",\n      \"name\": \"7a598452-8f61-4a4b-80ae-f7e88c1dabdb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:26:45.856883319Z\",\n      \"endTime\": \"2026-08-11T23:50:46.49113577Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lom43tvt8erhq92rpfrdqo0hb0ar9 still exists (deletion dispatched at 2026-08-11T23:26:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e89972490e56d6134e3fce8340fa2cadf09fbe33d50fcac40213b27dfc7dbe91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"72740c3b22addb3efec6e29c0fa9fd3c1b919caed53adc3f828532ab13efcce8","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c522b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-g8lc5d79v57s\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/576f799b-1706-4f8f-98d7-742350600ecc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/576f799b-1706-4f8f-98d7-742350600ecc\",\n      \"name\": \"576f799b-1706-4f8f-98d7-742350600ecc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:24:54.573326029Z\",\n      \"endTime\": \"2026-08-11T23:48:56.470952982Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lnu3cjs9iacscf1h6mgtj3vaeb9gh still exists (deletion dispatched at 2026-08-11T23:24:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001c522b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-g8lc5d79v57s\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/576f799b-1706-4f8f-98d7-742350600ecc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/576f799b-1706-4f8f-98d7-742350600ecc\",\n      \"name\": \"576f799b-1706-4f8f-98d7-742350600ecc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:24:54.573326029Z\",\n      \"endTime\": \"2026-08-11T23:48:56.470952982Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lnu3cjs9iacscf1h6mgtj3vaeb9gh still exists (deletion dispatched at 2026-08-11T23:24:55Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b936783c2d0b2a9d4ffabaeda14053cc9fef881b92be87e0e230002db2affbaf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ff8a9349d704dc9227105e1175a091acb7ab064986e6dc20c017f309540c1308","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2087313155679064064","occurred_at":"2026-08-11T23:00:11Z","failed_at":"e2e","signature_id":"4f96f13ee7bcfb7c4166ca2d7e01fd60eb9a279cf33432604877e1ffde66af21","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0024d60f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-6p8jn7dwhrl7\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/033bebfb-a5d4-4087-b904-0f9ac77a657e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/033bebfb-a5d4-4087-b904-0f9ac77a657e\",\n      \"name\": \"033bebfb-a5d4-4087-b904-0f9ac77a657e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:23:22.668117062Z\",\n      \"endTime\": \"2026-08-11T23:47:27.276194712Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lnu544pda59o3s0h1v97sl8v9d5je still exists (deletion dispatched at 2026-08-11T23:23:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc0024d60f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-6p8jn7dwhrl7\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/033bebfb-a5d4-4087-b904-0f9ac77a657e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/033bebfb-a5d4-4087-b904-0f9ac77a657e\",\n      \"name\": \"033bebfb-a5d4-4087-b904-0f9ac77a657e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-11T23:23:22.668117062Z\",\n      \"endTime\": \"2026-08-11T23:47:27.276194712Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s4lnu544pda59o3s0h1v97sl8v9d5je still exists (deletion dispatched at 2026-08-11T23:23:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":32,"failure_pattern_ids":["6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","7c6699de93ea43e4fe2694117b85c935fcdda27f6875f4ba53466c9c1cc25e4f","a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","b936783c2d0b2a9d4ffabaeda14053cc9fef881b92be87e0e230002db2affbaf","d6ec87c7d972149dae5ab13de0394af8c0dc25683fd32e5ecca933f42f5cfc45"],"matched_occurrences":32,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2087261751195209728","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-11T19:35:42Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2087197913452122112","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-11T15:22:15Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2086996063331291136","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-11T02:00:10Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}