{"meta":{"date":"2026-07-20","timezone":"UTC","generated_at":"2026-08-27T23:01:32Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":14,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":8,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-20T23:12:05Z"},"failed_at":["e2e"],"failed_test_count":45,"occurrences":[{"row_id":"0713faa2e196bf3ac61c722a32f12817c2706ebae3a62a682ec0f2513e01a6b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"265b6a0676836af4e807aef8fed5b704c3947e826ca78ceeeb8620193b2499ca","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:312]: failed to create HCP cluster np-noedge-gcr48x\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e687a0\u003e: \n    failed to create HCP cluster np-noedge-gcr48x: failed waiting for cluster=\"np-noedge-gcr48x\" in resourcegroup=\"rg-np-noedge-gcr48x-zbmq2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14211a29-e9a6-462f-971d-3a4e45b95ee1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14211a29-e9a6-462f-971d-3a4e45b95ee1\",\n      \"name\": \"14211a29-e9a6-462f-971d-3a4e45b95ee1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:17:31.297065785Z\",\n      \"endTime\": \"2026-07-21T00:36:37.107842263Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:312]: failed to create HCP cluster np-noedge-gcr48x\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e687a0\u003e: \n    failed to create HCP cluster np-noedge-gcr48x: failed waiting for cluster=\"np-noedge-gcr48x\" in resourcegroup=\"rg-np-noedge-gcr48x-zbmq2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14211a29-e9a6-462f-971d-3a4e45b95ee1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14211a29-e9a6-462f-971d-3a4e45b95ee1\",\n      \"name\": \"14211a29-e9a6-462f-971d-3a4e45b95ee1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:17:31.297065785Z\",\n      \"endTime\": \"2026-07-21T00:36:37.107842263Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"073ec27f66479a36e5d6dedb64952891ca8f66b14bc7436d0893b505febb2555","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"85d723b8b2acab4cc8c7772a3615d97b7b9eb49c6db84efcec9a2f051d1a8f91","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-btk2zg\" with version 4.22.4 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f208e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-btk2zg: failed waiting for cluster=\"cluster-zstream-4-22-btk2zg\" in resourcegroup=\"rg-zstream-upgrade-4-22-xs4gl6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14681ece-f5f2-45b9-8d66-0fe3b46e17ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14681ece-f5f2-45b9-8d66-0fe3b46e17ab\",\n      \"name\": \"14681ece-f5f2-45b9-8d66-0fe3b46e17ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:51:50.176402512Z\",\n      \"endTime\": \"2026-07-21T00:10:56.149266938Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-btk2zg\" with version 4.22.4 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f208e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-btk2zg: failed waiting for cluster=\"cluster-zstream-4-22-btk2zg\" in resourcegroup=\"rg-zstream-upgrade-4-22-xs4gl6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14681ece-f5f2-45b9-8d66-0fe3b46e17ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/14681ece-f5f2-45b9-8d66-0fe3b46e17ab\",\n      \"name\": \"14681ece-f5f2-45b9-8d66-0fe3b46e17ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:51:50.176402512Z\",\n      \"endTime\": \"2026-07-21T00:10:56.149266938Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"101c2b63fc73df451eb2fb02e1e9926c47fbdf7c74b5974c91d841f7b855a7e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"03ccea68e6a8b694dcafc46c74a57eef305bc4ba4bf6c778ce56d2fc61dd57d5","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a67ae0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-4qrkxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5084db0d-3e6b-41e7-a0c3-5104a663ded1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5084db0d-3e6b-41e7-a0c3-5104a663ded1\",\n      \"name\": \"5084db0d-3e6b-41e7-a0c3-5104a663ded1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:13:34.291625253Z\",\n      \"endTime\": \"2026-07-21T00:32:37.715507635Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a67ae0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-4qrkxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5084db0d-3e6b-41e7-a0c3-5104a663ded1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5084db0d-3e6b-41e7-a0c3-5104a663ded1\",\n      \"name\": \"5084db0d-3e6b-41e7-a0c3-5104a663ded1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:13:34.291625253Z\",\n      \"endTime\": \"2026-07-21T00:32:37.715507635Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1202ecff97f50c91c03c596111e572d002a89ad7bc1ac4535c54276b14a5ea7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"889ca6784d0531d5479e0c6017f00dd044e23f2c023d0791673f40a0192a6ff1","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7a620\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-jtl59z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a17de45-1712-45e7-ba26-a82ec08c105e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a17de45-1712-45e7-ba26-a82ec08c105e\",\n      \"name\": \"4a17de45-1712-45e7-ba26-a82ec08c105e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:13:36.904106278Z\",\n      \"endTime\": \"2026-07-21T00:32:37.819231401Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7a620\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-jtl59z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a17de45-1712-45e7-ba26-a82ec08c105e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a17de45-1712-45e7-ba26-a82ec08c105e\",\n      \"name\": \"4a17de45-1712-45e7-ba26-a82ec08c105e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:13:36.904106278Z\",\n      \"endTime\": \"2026-07-21T00:32:37.819231401Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"16a5725ea3718a0637b4e8edbdff058129ab38f5a92c1fbfed191955a4a08836","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"db7264e513dadaf676a31f0d7b1ab194ebcc3bd6cb073e2000a43d29e1527cb8","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-lgpjr8 with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001514a00\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-lgpjr8: failed waiting for cluster=\"np-version-upgrade-cluster-lgpjr8\" in resourcegroup=\"rg-np-version-upgrade-lgpjr8-6kng2b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/153eb99d-6d8d-4dab-92c6-67355312b7ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/153eb99d-6d8d-4dab-92c6-67355312b7ed\",\n      \"name\": \"153eb99d-6d8d-4dab-92c6-67355312b7ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:36:51.729825177Z\",\n      \"endTime\": \"2026-07-21T00:55:57.557090323Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-lgpjr8 with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001514a00\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-lgpjr8: failed waiting for cluster=\"np-version-upgrade-cluster-lgpjr8\" in resourcegroup=\"rg-np-version-upgrade-lgpjr8-6kng2b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/153eb99d-6d8d-4dab-92c6-67355312b7ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/153eb99d-6d8d-4dab-92c6-67355312b7ed\",\n      \"name\": \"153eb99d-6d8d-4dab-92c6-67355312b7ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:36:51.729825177Z\",\n      \"endTime\": \"2026-07-21T00:55:57.557090323Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"16be9153e16dddc88e02c096264218f46ca0da6b0f2d8b63d9cd90c25601da0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"73b3a4f6df42183726a6b451d6efe754a40a2b27033d5777a7d97226fb134c0f","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098e220\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-zxn7nn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/95d9a7df-2a4e-4f84-a966-403ab91ea89a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/95d9a7df-2a4e-4f84-a966-403ab91ea89a\",\n      \"name\": \"95d9a7df-2a4e-4f84-a966-403ab91ea89a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:15:37.125576763Z\",\n      \"endTime\": \"2026-07-21T00:34:37.38346775Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098e220\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-zxn7nn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/95d9a7df-2a4e-4f84-a966-403ab91ea89a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/95d9a7df-2a4e-4f84-a966-403ab91ea89a\",\n      \"name\": \"95d9a7df-2a4e-4f84-a966-403ab91ea89a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:15:37.125576763Z\",\n      \"endTime\": \"2026-07-21T00:34:37.38346775Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1de1b7ef330f41f93402c640133a1656c8a2e45a0310e80859ab7908f50e5972","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"34aa37716134afd6f7524c995d93155893fa8044ea2c34b3c11a938895a4451c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-21-mvtdpw\" with version 4.21.24 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00148c160\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-mvtdpw: failed waiting for cluster=\"cluster-zstream-4-21-mvtdpw\" in resourcegroup=\"rg-zstream-upgrade-4-21-fvlw2h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3e9e4860-a377-4798-951c-c4fb1265c36e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3e9e4860-a377-4798-951c-c4fb1265c36e\",\n      \"name\": \"3e9e4860-a377-4798-951c-c4fb1265c36e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:31.385489021Z\",\n      \"endTime\": \"2026-07-21T00:09:36.292394847Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-21-mvtdpw\" with version 4.21.24 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00148c160\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-mvtdpw: failed waiting for cluster=\"cluster-zstream-4-21-mvtdpw\" in resourcegroup=\"rg-zstream-upgrade-4-21-fvlw2h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3e9e4860-a377-4798-951c-c4fb1265c36e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3e9e4860-a377-4798-951c-c4fb1265c36e\",\n      \"name\": \"3e9e4860-a377-4798-951c-c4fb1265c36e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:31.385489021Z\",\n      \"endTime\": \"2026-07-21T00:09:36.292394847Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2684551a67bc225957bab5311aa1d0fbbb976b240473fe9d310ed758217d2499","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"bd236ac6ec5794320c2066b92f8aea4dc8c6219e3dcf658f4a63ea1b65873bce","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc2680\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6khn4q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/94d788ec-189e-4362-a3d2-cec0d3cdf922\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/94d788ec-189e-4362-a3d2-cec0d3cdf922\",\n      \"name\": \"94d788ec-189e-4362-a3d2-cec0d3cdf922\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:53:24.790797117Z\",\n      \"endTime\": \"2026-07-21T00:12:35.891697968Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc2680\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6khn4q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/94d788ec-189e-4362-a3d2-cec0d3cdf922\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/94d788ec-189e-4362-a3d2-cec0d3cdf922\",\n      \"name\": \"94d788ec-189e-4362-a3d2-cec0d3cdf922\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:53:24.790797117Z\",\n      \"endTime\": \"2026-07-21T00:12:35.891697968Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"270cb6e20ba90b93bc3e0f9a21120c8b7c58e6d3a4d59b94f992fc55ce244b60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"a47a9bdda6418eeb4e80459a24d7f7ea8445dbc0f1d211229bf30769944a7c63","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-zg2msw/cluster-candidate-4-22-chxf97 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c86c80\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-chxf97: failed waiting for cluster=\"cluster-candidate-4-22-chxf97\" in resourcegroup=\"rg-candidate-4-22-zg2msw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c30d3263-35fe-4461-887e-bbf69801993f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c30d3263-35fe-4461-887e-bbf69801993f\",\n      \"name\": \"c30d3263-35fe-4461-887e-bbf69801993f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:33.888830998Z\",\n      \"endTime\": \"2026-07-21T00:09:35.895668016Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-zg2msw/cluster-candidate-4-22-chxf97 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c86c80\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-chxf97: failed waiting for cluster=\"cluster-candidate-4-22-chxf97\" in resourcegroup=\"rg-candidate-4-22-zg2msw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c30d3263-35fe-4461-887e-bbf69801993f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c30d3263-35fe-4461-887e-bbf69801993f\",\n      \"name\": \"c30d3263-35fe-4461-887e-bbf69801993f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:33.888830998Z\",\n      \"endTime\": \"2026-07-21T00:09:35.895668016Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"32c0c382cf2ed009db6d3f054bfb48f11e88f57298dd2e82c5c30b9ece5af67b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"b10be977c5fd6622b4608d477b0a9cd716d109a55379e9dca73b913ef5093aa4","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015e0d60\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pvvc56\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/bd62d79a-cdbf-4506-b158-5b029d4a0f93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/bd62d79a-cdbf-4506-b158-5b029d4a0f93\",\n      \"name\": \"bd62d79a-cdbf-4506-b158-5b029d4a0f93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:09.612127418Z\",\n      \"endTime\": \"2026-07-21T00:09:16.010730105Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015e0d60\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-pvvc56\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/bd62d79a-cdbf-4506-b158-5b029d4a0f93\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/bd62d79a-cdbf-4506-b158-5b029d4a0f93\",\n      \"name\": \"bd62d79a-cdbf-4506-b158-5b029d4a0f93\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:09.612127418Z\",\n      \"endTime\": \"2026-07-21T00:09:16.010730105Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3abab3978b944e0dc12f18d781a72802dfbe577b0038f875cc32deca3b23a298","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"d9be2042771d4fcf101626ec2a5d59b4d10f0b7f28aee5d84055a7ea7d86373b","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e281c0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-p885n9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ce6ed49a-c2cd-4f6b-970a-c991b4923c0c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ce6ed49a-c2cd-4f6b-970a-c991b4923c0c\",\n      \"name\": \"ce6ed49a-c2cd-4f6b-970a-c991b4923c0c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:36:43.253166262Z\",\n      \"endTime\": \"2026-07-21T00:55:47.483495644Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e281c0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-p885n9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ce6ed49a-c2cd-4f6b-970a-c991b4923c0c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ce6ed49a-c2cd-4f6b-970a-c991b4923c0c\",\n      \"name\": \"ce6ed49a-c2cd-4f6b-970a-c991b4923c0c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:36:43.253166262Z\",\n      \"endTime\": \"2026-07-21T00:55:47.483495644Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"41063fee7eca24175eea3d7822ec59586f36c8a14b5d4c1901f5baaac785006e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"37ae3271f611d5ae0c32caff231fa1238016e661b29388af92b05d3e07cb3336","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-dkbjrb\" with version 4.20.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098a860\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-dkbjrb: failed waiting for cluster=\"cluster-zstream-4-20-dkbjrb\" in resourcegroup=\"rg-zstream-upgrade-4-20-2nmjxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/da2e092d-f95f-491f-a903-7f3eea4bbef8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/da2e092d-f95f-491f-a903-7f3eea4bbef8\",\n      \"name\": \"da2e092d-f95f-491f-a903-7f3eea4bbef8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:35.181363652Z\",\n      \"endTime\": \"2026-07-21T00:09:35.893935187Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-dkbjrb\" with version 4.20.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098a860\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-dkbjrb: failed waiting for cluster=\"cluster-zstream-4-20-dkbjrb\" in resourcegroup=\"rg-zstream-upgrade-4-20-2nmjxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/da2e092d-f95f-491f-a903-7f3eea4bbef8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/da2e092d-f95f-491f-a903-7f3eea4bbef8\",\n      \"name\": \"da2e092d-f95f-491f-a903-7f3eea4bbef8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:35.181363652Z\",\n      \"endTime\": \"2026-07-21T00:09:35.893935187Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4409375e1adf136d3e618ac7156a1bae2cd771c4102e710de039f086afeaecf7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"125e1eb21cff6d94a185f310953f766c546d0091298342f5e0d3b2b8f11d9ca3","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c73aa0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-wbrpm7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/810fd8dc-75cc-46e7-9935-40b1c132cd65\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/810fd8dc-75cc-46e7-9935-40b1c132cd65\",\n      \"name\": \"810fd8dc-75cc-46e7-9935-40b1c132cd65\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:20.299583025Z\",\n      \"endTime\": \"2026-07-20T23:46:34.265720908Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c73aa0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-wbrpm7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/810fd8dc-75cc-46e7-9935-40b1c132cd65\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/810fd8dc-75cc-46e7-9935-40b1c132cd65\",\n      \"name\": \"810fd8dc-75cc-46e7-9935-40b1c132cd65\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:20.299583025Z\",\n      \"endTime\": \"2026-07-20T23:46:34.265720908Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4f058f29a173afc59ccc48add0aa7bee23297454a87c3adf7ded24b48d9251c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"3f1f342ff7f5755b4c967ba4e5516a63554ba4129d837f57feabc2e71f247ace","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-20-jx4gz7/cluster-candidate-4-20-7xw8dx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c5f000\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-7xw8dx: failed waiting for cluster=\"cluster-candidate-4-20-7xw8dx\" in resourcegroup=\"rg-candidate-4-20-jx4gz7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8aefc8b8-8aca-4959-92ef-4a3aabb46376\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8aefc8b8-8aca-4959-92ef-4a3aabb46376\",\n      \"name\": \"8aefc8b8-8aca-4959-92ef-4a3aabb46376\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:52:12.967428063Z\",\n      \"endTime\": \"2026-07-21T00:11:16.713261936Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-20-jx4gz7/cluster-candidate-4-20-7xw8dx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c5f000\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-7xw8dx: failed waiting for cluster=\"cluster-candidate-4-20-7xw8dx\" in resourcegroup=\"rg-candidate-4-20-jx4gz7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8aefc8b8-8aca-4959-92ef-4a3aabb46376\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8aefc8b8-8aca-4959-92ef-4a3aabb46376\",\n      \"name\": \"8aefc8b8-8aca-4959-92ef-4a3aabb46376\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:52:12.967428063Z\",\n      \"endTime\": \"2026-07-21T00:11:16.713261936Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5060eb4eaf94bdc2443dba959e1c2431f335c8562ba40d10cef403c17b6d5f42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"82952c372d25be52bdb5f896a7d0c7fd134f1f78b3671581993ea57d9111643c","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-7d4pkq with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00046d700\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-7d4pkq: failed waiting for cluster=\"np-version-upgrade-cluster-7d4pkq\" in resourcegroup=\"rg-np-version-upgrade-7d4pkq-v625bd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7733647b-dd05-400c-8ba7-8196eeb49794\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7733647b-dd05-400c-8ba7-8196eeb49794\",\n      \"name\": \"7733647b-dd05-400c-8ba7-8196eeb49794\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:17:39.55384181Z\",\n      \"endTime\": \"2026-07-21T00:36:57.108773159Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-7d4pkq with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00046d700\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-7d4pkq: failed waiting for cluster=\"np-version-upgrade-cluster-7d4pkq\" in resourcegroup=\"rg-np-version-upgrade-7d4pkq-v625bd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7733647b-dd05-400c-8ba7-8196eeb49794\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7733647b-dd05-400c-8ba7-8196eeb49794\",\n      \"name\": \"7733647b-dd05-400c-8ba7-8196eeb49794\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:17:39.55384181Z\",\n      \"endTime\": \"2026-07-21T00:36:57.108773159Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"52462dd8ed95adf6ae6213f2852d932599f0f40514563b3a71a121ae493bd3be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"7a0134fabd9660f00a24a029b6933db1038908cb4142d60247d52db5de6de2f6","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-6clfvk with version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b823e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-6clfvk: failed waiting for cluster=\"np-version-upgrade-cluster-6clfvk\" in resourcegroup=\"rg-np-version-upgrade-6clfvk-bbn9b6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4b55f620-ed87-4313-b1bb-6323b9bf092a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4b55f620-ed87-4313-b1bb-6323b9bf092a\",\n      \"name\": \"4b55f620-ed87-4313-b1bb-6323b9bf092a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:13:39.321054413Z\",\n      \"endTime\": \"2026-07-21T00:32:47.18921824Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-6clfvk with version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b823e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-6clfvk: failed waiting for cluster=\"np-version-upgrade-cluster-6clfvk\" in resourcegroup=\"rg-np-version-upgrade-6clfvk-bbn9b6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4b55f620-ed87-4313-b1bb-6323b9bf092a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4b55f620-ed87-4313-b1bb-6323b9bf092a\",\n      \"name\": \"4b55f620-ed87-4313-b1bb-6323b9bf092a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:13:39.321054413Z\",\n      \"endTime\": \"2026-07-21T00:32:47.18921824Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"586ca517bf17820d9681b8cd351171aa5543fc61985a68236565f52f675af5c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"9eb4a664b9a274d1afc487ab37651ae198bf31417272ecc1061bd959bb65e625","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a22980\u003e: \n    failed to create HCP cluster ea-listmz5z29: failed waiting for cluster=\"ea-listmz5z29\" in resourcegroup=\"ea-list-mlwjs9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0dbffcd5-faaa-4add-82fb-7df60b416cce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0dbffcd5-faaa-4add-82fb-7df60b416cce\",\n      \"name\": \"0dbffcd5-faaa-4add-82fb-7df60b416cce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:53:09.285772804Z\",\n      \"endTime\": \"2026-07-21T00:12:16.021816887Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a22980\u003e: \n    failed to create HCP cluster ea-listmz5z29: failed waiting for cluster=\"ea-listmz5z29\" in resourcegroup=\"ea-list-mlwjs9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0dbffcd5-faaa-4add-82fb-7df60b416cce\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0dbffcd5-faaa-4add-82fb-7df60b416cce\",\n      \"name\": \"0dbffcd5-faaa-4add-82fb-7df60b416cce\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:53:09.285772804Z\",\n      \"endTime\": \"2026-07-21T00:12:16.021816887Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"632bba3a84aea614be71f5792ec5f9e70aa501ede9bdce32110e5109437b6c62","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"ad5b44c12212f5532334b5a7018294db4764bc67345d1e6f4be50920237aac9e","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000706440\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-zvmc8f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/988235b5-720f-487c-8072-3201187fba27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/988235b5-720f-487c-8072-3201187fba27\",\n      \"name\": \"988235b5-720f-487c-8072-3201187fba27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:16:25.886659508Z\",\n      \"endTime\": \"2026-07-21T00:35:37.084005503Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000706440\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-zvmc8f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/988235b5-720f-487c-8072-3201187fba27\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/988235b5-720f-487c-8072-3201187fba27\",\n      \"name\": \"988235b5-720f-487c-8072-3201187fba27\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:16:25.886659508Z\",\n      \"endTime\": \"2026-07-21T00:35:37.084005503Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"717d573b6ebd7b28eaecd76a29d99b44355a8dea7b9332597695e5c41f871ab5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"1711f5728df2e5c297c08418880e01c60aea1816ee258b02b00c95efad4f0a29","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f56c60\u003e: \n    failed to create HCP cluster np-dg-minor-948grk: failed waiting for cluster=\"np-dg-minor-948grk\" in resourcegroup=\"rg-np-dg-minor-948grk-d2ndbs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5dd650d3-6e39-4696-be62-516c9def9199\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5dd650d3-6e39-4696-be62-516c9def9199\",\n      \"name\": \"5dd650d3-6e39-4696-be62-516c9def9199\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:15:37.246546157Z\",\n      \"endTime\": \"2026-07-21T00:34:37.47529642Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f56c60\u003e: \n    failed to create HCP cluster np-dg-minor-948grk: failed waiting for cluster=\"np-dg-minor-948grk\" in resourcegroup=\"rg-np-dg-minor-948grk-d2ndbs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5dd650d3-6e39-4696-be62-516c9def9199\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5dd650d3-6e39-4696-be62-516c9def9199\",\n      \"name\": \"5dd650d3-6e39-4696-be62-516c9def9199\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:15:37.246546157Z\",\n      \"endTime\": \"2026-07-21T00:34:37.47529642Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"722c4c7dc40645cca5333a8f4b8258b58353351b8dcc99c909b6ef5b12497f9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"a41a9dd0285ca9f12cb560711358249e2025fbe7cf12cf82a42396190c9042c9","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fee390\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-d2dn29\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-d2dn29, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fee390\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-d2dn29\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-d2dn29, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-0-vxt256, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: failed waiting for cluster=\"cluster-nightly-5-0-vxt256\" in resourcegroup=\"rg-nightly-5-0-cpmsgk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: context deadline exceeded"}},{"row_id":"752e9e5826e972f9bd6d4834001ba25ff93fa670e9c1d63717ee3cc4e1e7fc5e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"e6fb9d50a85afd951185ada7264e12cbc062b6c2d0863a724ca8b0bdb5d59230","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-9tdk5f\" with version 4.19.37 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de1580\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-9tdk5f: failed waiting for cluster=\"cluster-zstream-4-19-9tdk5f\" in resourcegroup=\"rg-zstream-upgrade-4-19-5xdq9f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a4a0278-31ba-43d1-9cce-357ca5cf2db9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a4a0278-31ba-43d1-9cce-357ca5cf2db9\",\n      \"name\": \"9a4a0278-31ba-43d1-9cce-357ca5cf2db9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:24.532800656Z\",\n      \"endTime\": \"2026-07-21T00:09:35.869682594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-9tdk5f\" with version 4.19.37 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000de1580\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-9tdk5f: failed waiting for cluster=\"cluster-zstream-4-19-9tdk5f\" in resourcegroup=\"rg-zstream-upgrade-4-19-5xdq9f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a4a0278-31ba-43d1-9cce-357ca5cf2db9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a4a0278-31ba-43d1-9cce-357ca5cf2db9\",\n      \"name\": \"9a4a0278-31ba-43d1-9cce-357ca5cf2db9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:24.532800656Z\",\n      \"endTime\": \"2026-07-21T00:09:35.869682594Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7969f0182832f70cb6b094b9f9c30b27f9a514828aaeb205331c71e9fe06825b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"075407059f3a5b3d581b963f2098a2be85da109dc5ca5fcc1562bfbb79dd101f","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:131]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00155ef00\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-l96fjf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-l96fjf/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Network/publicIPAddresses (2022-07-01)' reported preflight validation errors. Tracking id is '3a84a790-e152-4d73-a198-bf0f9d6b6d4a'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"ResourceCountExceedsLimitDueToTemplate\",\n            \"message\": \"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 300 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 300 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\",\n            \"details\": []\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:131]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00155ef00\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-l96fjf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-l96fjf/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Network/publicIPAddresses (2022-07-01)' reported preflight validation errors. Tracking id is '3a84a790-e152-4d73-a198-bf0f9d6b6d4a'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"ResourceCountExceedsLimitDueToTemplate\",\n            \"message\": \"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 300 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 300 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\",\n            \"details\": []\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b43e7c6c150476d7400be2924e381be5a30b739ff1a4de2439b70edbc0e574e4","failure_pattern":"ERROR CODE: InvalidTemplateDeployment; provider Microsoft.Network","search_query":"ERROR CODE: InvalidTemplateDeployment"}},{"row_id":"7a80431c929766578b3c7ff54629791a4980d8397925a3b9197a7bf21b09155a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"5311ecd3697fc9f18c939f8dd5620e37778a9eb72c6a9cd3f55aa32d8cb91a66","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cb8300\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-jztjj4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47296f8d-600b-40e8-b68e-81a15af3b0a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47296f8d-600b-40e8-b68e-81a15af3b0a5\",\n      \"name\": \"47296f8d-600b-40e8-b68e-81a15af3b0a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:13:25.428084669Z\",\n      \"endTime\": \"2026-07-21T00:32:37.0533164Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cb8300\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-jztjj4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47296f8d-600b-40e8-b68e-81a15af3b0a5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/47296f8d-600b-40e8-b68e-81a15af3b0a5\",\n      \"name\": \"47296f8d-600b-40e8-b68e-81a15af3b0a5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:13:25.428084669Z\",\n      \"endTime\": \"2026-07-21T00:32:37.0533164Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"923bc8ab123e48204201b89c1e056e4caa7d1054d43285c88a53bad809401866","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"91676d4a01ae25233828ea04f96c2ac2902ecb8011e42564e12278ad78aa1aed","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e28ca0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-dxs24s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e6e549d7-9327-473c-85da-efe60d6b427d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e6e549d7-9327-473c-85da-efe60d6b427d\",\n      \"name\": \"e6e549d7-9327-473c-85da-efe60d6b427d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:51:44.983259602Z\",\n      \"endTime\": \"2026-07-21T00:10:56.043526282Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e28ca0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-dxs24s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e6e549d7-9327-473c-85da-efe60d6b427d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e6e549d7-9327-473c-85da-efe60d6b427d\",\n      \"name\": \"e6e549d7-9327-473c-85da-efe60d6b427d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:51:44.983259602Z\",\n      \"endTime\": \"2026-07-21T00:10:56.043526282Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"93362d88a5c2d4eb75db0259d9c52e784ef0bcd6c57ad4c8b84f3cc4972ae032","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"14f34584497f527d1fd254e155a719f6a47e264ec4152faf4578d0921b44b781","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e086a0\u003e: \n    failed to create HCP cluster np-downgrade-kfxhrl: failed waiting for cluster=\"np-downgrade-kfxhrl\" in resourcegroup=\"rg-np-downgrade-kfxhrl-2h659z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fae6bfc3-5e71-499a-93d1-a47be79e943a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fae6bfc3-5e71-499a-93d1-a47be79e943a\",\n      \"name\": \"fae6bfc3-5e71-499a-93d1-a47be79e943a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:15:07.341508774Z\",\n      \"endTime\": \"2026-07-21T00:34:07.168232803Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e086a0\u003e: \n    failed to create HCP cluster np-downgrade-kfxhrl: failed waiting for cluster=\"np-downgrade-kfxhrl\" in resourcegroup=\"rg-np-downgrade-kfxhrl-2h659z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fae6bfc3-5e71-499a-93d1-a47be79e943a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fae6bfc3-5e71-499a-93d1-a47be79e943a\",\n      \"name\": \"fae6bfc3-5e71-499a-93d1-a47be79e943a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:15:07.341508774Z\",\n      \"endTime\": \"2026-07-21T00:34:07.168232803Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"96fea8c63193750cd74a3264fa281a5c44b9803415e9eb81971c7311fcee9777","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"64925d571062ab351e4970f8614937952c489f1e50feda8e46999ee55fc6366e","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1eba0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-6vq7tp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/05a64bb6-f871-4528-ac47-c25653cada42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/05a64bb6-f871-4528-ac47-c25653cada42\",\n      \"name\": \"05a64bb6-f871-4528-ac47-c25653cada42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:20.134166987Z\",\n      \"endTime\": \"2026-07-20T23:46:34.547971528Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1eba0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-6vq7tp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/05a64bb6-f871-4528-ac47-c25653cada42\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/05a64bb6-f871-4528-ac47-c25653cada42\",\n      \"name\": \"05a64bb6-f871-4528-ac47-c25653cada42\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:20.134166987Z\",\n      \"endTime\": \"2026-07-20T23:46:34.547971528Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"99fb0b54a750036842fcb7e9ed5b7de82103ca80c3f0e74fa254585d0337cca0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"9e2a681885d9c5d7b3bd72f6fa071a05b4f3d3f1de5a426f90bda626d847aaaa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"65e1e9a0baaa34ed8a9245e38e5e42f14d1ec4eaf26d25799fe41e9b2ddef4b2","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00024eb10\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fc5f5768-125a-45ca-a834-7a33b15f8899\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fc5f5768-125a-45ca-a834-7a33b15f8899\",\n      \"name\": \"fc5f5768-125a-45ca-a834-7a33b15f8899\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:28:31.724021061Z\",\n      \"endTime\": \"2026-07-20T23:47:34.550898973Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00024eb10\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fc5f5768-125a-45ca-a834-7a33b15f8899\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fc5f5768-125a-45ca-a834-7a33b15f8899\",\n      \"name\": \"fc5f5768-125a-45ca-a834-7a33b15f8899\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:28:31.724021061Z\",\n      \"endTime\": \"2026-07-20T23:47:34.550898973Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a26120e055529e440d746aa95fc5ec455b51e67afa10f91b1741336522dfca76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"db9139d39257b8cc2ffd82462d8fde2cffc5d888dca00eefbb3a3f6ce61a43e9","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a36220\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-svbm77\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/2b4cc035-2489-4402-8e4b-b27e3e569b77\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/2b4cc035-2489-4402-8e4b-b27e3e569b77\",\n      \"name\": \"2b4cc035-2489-4402-8e4b-b27e3e569b77\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:54:26.376607075Z\",\n      \"endTime\": \"2026-07-21T00:13:35.916969396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a36220\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-svbm77\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/2b4cc035-2489-4402-8e4b-b27e3e569b77\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/2b4cc035-2489-4402-8e4b-b27e3e569b77\",\n      \"name\": \"2b4cc035-2489-4402-8e4b-b27e3e569b77\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:54:26.376607075Z\",\n      \"endTime\": \"2026-07-21T00:13:35.916969396Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a37bb0b58e1761ac98ed5fbb77d6bd69644282365b7dc9ea428e4d329c7564a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"8049eb3eb04c5531e21091257e373deb4dd2c7db96f486e3da914e577f9aec88","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a95080\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-284lhg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1bf09854-b364-4307-aa58-06efc6669be2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1bf09854-b364-4307-aa58-06efc6669be2\",\n      \"name\": \"1bf09854-b364-4307-aa58-06efc6669be2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:19.686753629Z\",\n      \"endTime\": \"2026-07-20T23:46:34.53808567Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a95080\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-284lhg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1bf09854-b364-4307-aa58-06efc6669be2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1bf09854-b364-4307-aa58-06efc6669be2\",\n      \"name\": \"1bf09854-b364-4307-aa58-06efc6669be2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:19.686753629Z\",\n      \"endTime\": \"2026-07-20T23:46:34.53808567Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a9a577bdda17713097df3ed823a159e253afdbf88d9498759827351a8bc79259","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"ad889dc89797757e46a01f3df244f03758d124e92a6f3cec41f95608b876877c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"d81a152b208949a1d6b2e1fe144bda9593c322e64c29e592658f13d1f358e86e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c29c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gm2sxx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7ab4f41d-5636-49c8-bc7d-221e53fd1451\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7ab4f41d-5636-49c8-bc7d-221e53fd1451\",\n      \"name\": \"7ab4f41d-5636-49c8-bc7d-221e53fd1451\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:15:51.339800606Z\",\n      \"endTime\": \"2026-07-21T00:35:07.082540317Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c29c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-gm2sxx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7ab4f41d-5636-49c8-bc7d-221e53fd1451\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7ab4f41d-5636-49c8-bc7d-221e53fd1451\",\n      \"name\": \"7ab4f41d-5636-49c8-bc7d-221e53fd1451\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:15:51.339800606Z\",\n      \"endTime\": \"2026-07-21T00:35:07.082540317Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb7cf9e14e4bddb34a99ee82b673c6a77c8c953a2b5d74f9f67ad60ac8d09c83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"3702e372a06a775b7ed3e266ebc6ae35c02f7531681a330d32c38e066d0dbf75","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009016c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-57pvp7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/68731108-ecd6-4d35-91ce-45f888679665\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/68731108-ecd6-4d35-91ce-45f888679665\",\n      \"name\": \"68731108-ecd6-4d35-91ce-45f888679665\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:18.798916736Z\",\n      \"endTime\": \"2026-07-20T23:46:34.278050405Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009016c0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-57pvp7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/68731108-ecd6-4d35-91ce-45f888679665\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/68731108-ecd6-4d35-91ce-45f888679665\",\n      \"name\": \"68731108-ecd6-4d35-91ce-45f888679665\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:18.798916736Z\",\n      \"endTime\": \"2026-07-20T23:46:34.278050405Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c4391d4940be5e9a03ddce459cc21df2bc20e66fc0ac4573997a9651bf14bcff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"4ec9b238739f8013c9d38e8bb0fd3a4a49a36f255a2ee65d8f03d1caa2b000a5","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9f3c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-47xxkw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8c0c597e-e571-4665-8ebb-68ebd42a574e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8c0c597e-e571-4665-8ebb-68ebd42a574e\",\n      \"name\": \"8c0c597e-e571-4665-8ebb-68ebd42a574e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:16:04.344384294Z\",\n      \"endTime\": \"2026-07-21T00:35:07.067692277Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9f3c0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-47xxkw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8c0c597e-e571-4665-8ebb-68ebd42a574e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8c0c597e-e571-4665-8ebb-68ebd42a574e\",\n      \"name\": \"8c0c597e-e571-4665-8ebb-68ebd42a574e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:16:04.344384294Z\",\n      \"endTime\": \"2026-07-21T00:35:07.067692277Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c530c6142e19c1a71508d67108ddc231f4b3151a5c5eca254a5dd4fa3e5ddbc0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"714ae85975cb35cbc648aaf48a47b60e3e3a910cae8cd699eabc8e961d796f0c","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e22240\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-k4rlzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/69814de1-7bfc-4593-85ab-d74e6cad3ef4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/69814de1-7bfc-4593-85ab-d74e6cad3ef4\",\n      \"name\": \"69814de1-7bfc-4593-85ab-d74e6cad3ef4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:29:03.175561473Z\",\n      \"endTime\": \"2026-07-20T23:48:14.452637462Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e22240\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-k4rlzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/69814de1-7bfc-4593-85ab-d74e6cad3ef4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/69814de1-7bfc-4593-85ab-d74e6cad3ef4\",\n      \"name\": \"69814de1-7bfc-4593-85ab-d74e6cad3ef4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:29:03.175561473Z\",\n      \"endTime\": \"2026-07-20T23:48:14.452637462Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c6473b049d1aa5dfc1d123d55237269751a48a91ddc406526d197e4a605106d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"4d4a0c495fde836e9b2267ae6dc30c2ced3ec30a939332cd344d4f3903963950","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001928380\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p2zzff\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/60f51ae9-cccd-4523-aabf-8a2f7e591bab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/60f51ae9-cccd-4523-aabf-8a2f7e591bab\",\n      \"name\": \"60f51ae9-cccd-4523-aabf-8a2f7e591bab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:13:21.077768725Z\",\n      \"endTime\": \"2026-07-21T00:32:37.33599024Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001928380\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-p2zzff\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/60f51ae9-cccd-4523-aabf-8a2f7e591bab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/60f51ae9-cccd-4523-aabf-8a2f7e591bab\",\n      \"name\": \"60f51ae9-cccd-4523-aabf-8a2f7e591bab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:13:21.077768725Z\",\n      \"endTime\": \"2026-07-21T00:32:37.33599024Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d330267d457a4abbcffd04848f9e66a99212d3baecd3c7ed0b34fd14cf329cb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"add34bd17b15b0e5f3baa3660324a51f7fc57c1aa4c40a95f7ba5f5d5a3af52d","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e264c0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-xfqmmd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90f81d48-56f6-4384-930a-ed2bd730dc6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90f81d48-56f6-4384-930a-ed2bd730dc6f\",\n      \"name\": \"90f81d48-56f6-4384-930a-ed2bd730dc6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:21.997343947Z\",\n      \"endTime\": \"2026-07-20T23:46:34.293460852Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e264c0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-xfqmmd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90f81d48-56f6-4384-930a-ed2bd730dc6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90f81d48-56f6-4384-930a-ed2bd730dc6f\",\n      \"name\": \"90f81d48-56f6-4384-930a-ed2bd730dc6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:21.997343947Z\",\n      \"endTime\": \"2026-07-20T23:46:34.293460852Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d3a2e7764921f683d8037a19b6d6099f81ce22202fbce1d862f7ab3a12da3b24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"81016f7db0c3e3dd63e6633c306735ea684491fabd3cca8d82769f62a3ddf33f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:181]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-w5htc7\" with install version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009f21a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-w5htc7: failed waiting for cluster=\"cp-ystream-upgrade-4-21-w5htc7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-pmtzgk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7f61abdb-048c-4804-bef0-d3df10817907\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7f61abdb-048c-4804-bef0-d3df10817907\",\n      \"name\": \"7f61abdb-048c-4804-bef0-d3df10817907\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:51:17.625121238Z\",\n      \"endTime\": \"2026-07-21T00:10:36.475521937Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:181]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-w5htc7\" with install version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009f21a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-w5htc7: failed waiting for cluster=\"cp-ystream-upgrade-4-21-w5htc7\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-pmtzgk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7f61abdb-048c-4804-bef0-d3df10817907\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7f61abdb-048c-4804-bef0-d3df10817907\",\n      \"name\": \"7f61abdb-048c-4804-bef0-d3df10817907\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:51:17.625121238Z\",\n      \"endTime\": \"2026-07-21T00:10:36.475521937Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d696e0b818b42e187ea5a3ab7a30aea957fe6d541c485dc9889f2e201056ec13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"d44f04715e3fc9b42e2b2b692914e2eae01d079b82aef9e4defbe36104eee6f3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-956kml/cluster-candidate-5-0-kp9g8t should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e98aa0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-kp9g8t: failed waiting for cluster=\"cluster-candidate-5-0-kp9g8t\" in resourcegroup=\"rg-candidate-5-0-956kml\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a55d137-4efa-4047-a368-8b3e575e0eec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a55d137-4efa-4047-a368-8b3e575e0eec\",\n      \"name\": \"4a55d137-4efa-4047-a368-8b3e575e0eec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:25.613264438Z\",\n      \"endTime\": \"2026-07-21T00:09:36.076795552Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-956kml/cluster-candidate-5-0-kp9g8t should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e98aa0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-kp9g8t: failed waiting for cluster=\"cluster-candidate-5-0-kp9g8t\" in resourcegroup=\"rg-candidate-5-0-956kml\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a55d137-4efa-4047-a368-8b3e575e0eec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4a55d137-4efa-4047-a368-8b3e575e0eec\",\n      \"name\": \"4a55d137-4efa-4047-a368-8b3e575e0eec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:25.613264438Z\",\n      \"endTime\": \"2026-07-21T00:09:36.076795552Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d860f3944dcb3b5ffe4cc1cbf7a1c089af9b14faf3bf41a066cc8ddc8d68b375","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"e1c181b0ec5dd4f32f696abf57f3897e3d3ccf2593d37f74a0000ca1f79f2764","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create HCP cluster np-skip-minor-xg2hgt at version 4.22.6\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9ed40\u003e: \n    failed to create HCP cluster np-skip-minor-xg2hgt: failed waiting for cluster=\"np-skip-minor-xg2hgt\" in resourcegroup=\"rg-np-skip-minor-xg2hgt-xnlznk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ebbb47f-36c2-4b3e-9a5b-a1f27da4b9ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ebbb47f-36c2-4b3e-9a5b-a1f27da4b9ef\",\n      \"name\": \"0ebbb47f-36c2-4b3e-9a5b-a1f27da4b9ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:14:06.59397692Z\",\n      \"endTime\": \"2026-07-21T00:33:07.499260544Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create HCP cluster np-skip-minor-xg2hgt at version 4.22.6\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9ed40\u003e: \n    failed to create HCP cluster np-skip-minor-xg2hgt: failed waiting for cluster=\"np-skip-minor-xg2hgt\" in resourcegroup=\"rg-np-skip-minor-xg2hgt-xnlznk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ebbb47f-36c2-4b3e-9a5b-a1f27da4b9ef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ebbb47f-36c2-4b3e-9a5b-a1f27da4b9ef\",\n      \"name\": \"0ebbb47f-36c2-4b3e-9a5b-a1f27da4b9ef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T00:14:06.59397692Z\",\n      \"endTime\": \"2026-07-21T00:33:07.499260544Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ed53f5ca4de9a68c89109e95d5a81e3c2dc2415b255414ddcac5b29a77677d68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"288cddf07006d2c3c590457909901d860be8b9ebdf2fbf927ec2e0d37decf14c","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c44180\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-5w6k65\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cd38e039-4112-483b-8393-033cdc84a47d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cd38e039-4112-483b-8393-033cdc84a47d\",\n      \"name\": \"cd38e039-4112-483b-8393-033cdc84a47d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:16.161321606Z\",\n      \"endTime\": \"2026-07-20T23:46:34.29581269Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c44180\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-5w6k65\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cd38e039-4112-483b-8393-033cdc84a47d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cd38e039-4112-483b-8393-033cdc84a47d\",\n      \"name\": \"cd38e039-4112-483b-8393-033cdc84a47d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:16.161321606Z\",\n      \"endTime\": \"2026-07-20T23:46:34.29581269Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f37f4c1b4b0171d2bf42c92bbbe4e65df42f7a22fa2a4e9cbf651f1691726d8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"89557bccb2b4df5db7d906bb711a4915dfe553406ec56e8185e7174548413e24","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-pt2hjb/cluster-candidate-4-21-2ll7bn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0a820\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-2ll7bn: failed waiting for cluster=\"cluster-candidate-4-21-2ll7bn\" in resourcegroup=\"rg-candidate-4-21-pt2hjb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7fc635fe-f006-451b-aefc-b2ffc01fb9b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7fc635fe-f006-451b-aefc-b2ffc01fb9b8\",\n      \"name\": \"7fc635fe-f006-451b-aefc-b2ffc01fb9b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:10.418970082Z\",\n      \"endTime\": \"2026-07-21T00:09:16.325493793Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-pt2hjb/cluster-candidate-4-21-2ll7bn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0a820\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-2ll7bn: failed waiting for cluster=\"cluster-candidate-4-21-2ll7bn\" in resourcegroup=\"rg-candidate-4-21-pt2hjb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7fc635fe-f006-451b-aefc-b2ffc01fb9b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7fc635fe-f006-451b-aefc-b2ffc01fb9b8\",\n      \"name\": \"7fc635fe-f006-451b-aefc-b2ffc01fb9b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:10.418970082Z\",\n      \"endTime\": \"2026-07-21T00:09:16.325493793Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f905c5f1f6ff0f6b8c18f16b4b741000373d418df05da50867738b5fc6e62a36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"554d1257a5d97b619b43073ef9e77730c3695a30dc7ed68381c9e299427ea0ea","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001484a00\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-7qdfln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b10c0065-7db4-441a-b557-aecb0bf8c76e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b10c0065-7db4-441a-b557-aecb0bf8c76e\",\n      \"name\": \"b10c0065-7db4-441a-b557-aecb0bf8c76e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:21.322451152Z\",\n      \"endTime\": \"2026-07-20T23:46:34.544383871Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001484a00\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-7qdfln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b10c0065-7db4-441a-b557-aecb0bf8c76e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b10c0065-7db4-441a-b557-aecb0bf8c76e\",\n      \"name\": \"b10c0065-7db4-441a-b557-aecb0bf8c76e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:27:21.322451152Z\",\n      \"endTime\": \"2026-07-20T23:46:34.544383871Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fcfcf63651d4311f3ec22e9547144cc983f101078e2d91cccaab82629e6340b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"6ac7afa86bc2d5e90600e0fa431ab0bf71a8ef325a6e37103c2002fc89a835b1","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db82a0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n542fg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/580fc606-4c6a-421a-85bd-2417037c3c7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/580fc606-4c6a-421a-85bd-2417037c3c7b\",\n      \"name\": \"580fc606-4c6a-421a-85bd-2417037c3c7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:30:13.298295724Z\",\n      \"endTime\": \"2026-07-20T23:49:14.484408565Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000db82a0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n542fg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/580fc606-4c6a-421a-85bd-2417037c3c7b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/580fc606-4c6a-421a-85bd-2417037c3c7b\",\n      \"name\": \"580fc606-4c6a-421a-85bd-2417037c3c7b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:30:13.298295724Z\",\n      \"endTime\": \"2026-07-20T23:49:14.484408565Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fe6b7d302d5ae8925373bbccd7bae7bf076fe42aaae4bd92b766174210747619","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079343619860336640","occurred_at":"2026-07-20T23:12:05Z","failed_at":"e2e","signature_id":"4194b8de2fe09b669dfd595668949649e6d0742b203f5663747d477a79d6c348","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c80980\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-wn7gpr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1f4cf303-79f1-4123-a109-479a19aeed98\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1f4cf303-79f1-4123-a109-479a19aeed98\",\n      \"name\": \"1f4cf303-79f1-4123-a109-479a19aeed98\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:17.685484658Z\",\n      \"endTime\": \"2026-07-21T00:09:36.162887848Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c80980\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-wn7gpr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1f4cf303-79f1-4123-a109-479a19aeed98\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1f4cf303-79f1-4123-a109-479a19aeed98\",\n      \"name\": \"1f4cf303-79f1-4123-a109-479a19aeed98\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T23:50:17.685484658Z\",\n      \"endTime\": \"2026-07-21T00:09:36.162887848Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":45,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","b43e7c6c150476d7400be2924e381be5a30b739ff1a4de2439b70edbc0e574e4","f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b"],"matched_occurrences":45,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2079340761836425216","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-20T23:00:44Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"49e40032825d770f79927e42d89417b362d3a059d55951db7c0f0300a68c2e59","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2079340761836425216","occurred_at":"2026-07-20T23:00:44Z","failed_at":"e2e","signature_id":"4127a9187d48928f29732c57d0737caa49f7885f5a800e49e26c0ea8b42fe648","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-cpmsgk/cluster-nightly-5-0-vxt256 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00189f830\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-vxt256, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: failed waiting for cluster=\"cluster-nightly-5-0-vxt256\" in resourcegroup=\"rg-nightly-5-0-cpmsgk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-cpmsgk/cluster-nightly-5-0-vxt256 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00189f830\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-vxt256, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: failed waiting for cluster=\"cluster-nightly-5-0-vxt256\" in resourcegroup=\"rg-nightly-5-0-cpmsgk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-0-vxt256, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: failed waiting for cluster=\"cluster-nightly-5-0-vxt256\" in resourcegroup=\"rg-nightly-5-0-cpmsgk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079317230851461120","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-20T21:27:14Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f2748fd8ae346cadf043edb89267d5ea8d9a5695dc7a8067ad7d35c1c955094d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079317230851461120","occurred_at":"2026-07-20T21:27:14Z","failed_at":"e2e","signature_id":"ed61138e00b6a5005e4f6c12099e2f0f17e5485a149e9e9870bbab9d9d102348","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a0b110\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-sample-app-7gqvj.apps.aro.cilium-cl.gb92.westeurope.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a0b110\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-sample-app-7gqvj.apps.aro.cilium-cl.gb92.westeurope.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079316471627911168","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-20T21:24:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079251753139703808","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-20T17:07:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079247639689302016","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-20T16:50:42Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-20T16:02:17Z"},"failed_at":["e2e"],"failed_test_count":45,"occurrences":[{"row_id":"08b61a419a53a7f7f08a4cc3866268fa9ba87cad33cf4ebc77c1215f73ceb947","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"d1b2d1543f80338156eab672e9aabc48a634a2e3aed6e7362f6aeae1408ad8a1","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010301a0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-hfrj7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/edef3ced-f4ba-4b55-ac59-e8f085ddfaf0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/edef3ced-f4ba-4b55-ac59-e8f085ddfaf0\",\n      \"name\": \"edef3ced-f4ba-4b55-ac59-e8f085ddfaf0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:15.649450825Z\",\n      \"endTime\": \"2026-07-20T16:37:17.0362895Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010301a0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-hfrj7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/edef3ced-f4ba-4b55-ac59-e8f085ddfaf0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/edef3ced-f4ba-4b55-ac59-e8f085ddfaf0\",\n      \"name\": \"edef3ced-f4ba-4b55-ac59-e8f085ddfaf0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:15.649450825Z\",\n      \"endTime\": \"2026-07-20T16:37:17.0362895Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0af0162fdec714ede36a01d864e3e2d56d28299803e3a794fec2d56c3fa390d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"435316384ebf44ef449ed059b08798d4179c73e4fec5c50dee33eaede524a944","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fd41a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-g4bhdc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7edda124-19b7-473c-9887-19f43cdb8aea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7edda124-19b7-473c-9887-19f43cdb8aea\",\n      \"name\": \"7edda124-19b7-473c-9887-19f43cdb8aea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:20:10.545223447Z\",\n      \"endTime\": \"2026-07-20T16:39:16.751711284Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fd41a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-g4bhdc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7edda124-19b7-473c-9887-19f43cdb8aea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7edda124-19b7-473c-9887-19f43cdb8aea\",\n      \"name\": \"7edda124-19b7-473c-9887-19f43cdb8aea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:20:10.545223447Z\",\n      \"endTime\": \"2026-07-20T16:39:16.751711284Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0b083785fa0744af37b6a1dceee8028929a2e5a968ab65b4588b59bb0a9eda17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"415f5b5973ba3f3ee85b298687a47fadedce7293bdc49d77aa6a71e464b88fc7","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8ae40\u003e: \n    failed to create HCP cluster np-dg-minor-5flzzg: failed waiting for cluster=\"np-dg-minor-5flzzg\" in resourcegroup=\"rg-np-dg-minor-5flzzg-pckfx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c0849783-5217-4af9-ae68-8d3950320c0e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c0849783-5217-4af9-ae68-8d3950320c0e\",\n      \"name\": \"c0849783-5217-4af9-ae68-8d3950320c0e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:45.816351028Z\",\n      \"endTime\": \"2026-07-20T18:10:49.661294684Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8ae40\u003e: \n    failed to create HCP cluster np-dg-minor-5flzzg: failed waiting for cluster=\"np-dg-minor-5flzzg\" in resourcegroup=\"rg-np-dg-minor-5flzzg-pckfx9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c0849783-5217-4af9-ae68-8d3950320c0e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c0849783-5217-4af9-ae68-8d3950320c0e\",\n      \"name\": \"c0849783-5217-4af9-ae68-8d3950320c0e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:45.816351028Z\",\n      \"endTime\": \"2026-07-20T18:10:49.661294684Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"122f26634877b18de1dc9f358faca07e7a8989d25f1a2b34d50a9bf546a99600","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"642426adbe17e096821e1059b47daeef68c4b1aff59c8cf307463e5911048a68","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0010e6cf0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/365c8668-5b79-47ca-a36e-146e244d8ba6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/365c8668-5b79-47ca-a36e-146e244d8ba6\",\n      \"name\": \"365c8668-5b79-47ca-a36e-146e244d8ba6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:19:19.453873297Z\",\n      \"endTime\": \"2026-07-20T16:38:26.729684039Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0010e6cf0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/365c8668-5b79-47ca-a36e-146e244d8ba6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/365c8668-5b79-47ca-a36e-146e244d8ba6\",\n      \"name\": \"365c8668-5b79-47ca-a36e-146e244d8ba6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:19:19.453873297Z\",\n      \"endTime\": \"2026-07-20T16:38:26.729684039Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1592f15918e8870c136a7b6fab3952ff98ddc8b6f613a59181e8a4766d5fb556","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"a8e03a7b26fd123e4dd6455e921075b3852c4e1e7487e4d00ead13c966b1f8bf","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00080a520\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-zbssbt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/277e8a14-8fa1-4d41-ba83-33fbe3ee356d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/277e8a14-8fa1-4d41-ba83-33fbe3ee356d\",\n      \"name\": \"277e8a14-8fa1-4d41-ba83-33fbe3ee356d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:04.688858269Z\",\n      \"endTime\": \"2026-07-20T16:37:17.040207063Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00080a520\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-zbssbt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/277e8a14-8fa1-4d41-ba83-33fbe3ee356d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/277e8a14-8fa1-4d41-ba83-33fbe3ee356d\",\n      \"name\": \"277e8a14-8fa1-4d41-ba83-33fbe3ee356d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:04.688858269Z\",\n      \"endTime\": \"2026-07-20T16:37:17.040207063Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"18ae98126b9ffcd2a9c0089ff2b5b3fa3476638c839185c893fea161ca55f394","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"0034558eead0b80d52546e7575197511515ec3d6fa978947e41fe56d1b8dc0ec","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-6dlm44\" with version 4.20.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9c180\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-6dlm44: failed waiting for cluster=\"cluster-zstream-4-20-6dlm44\" in resourcegroup=\"rg-zstream-upgrade-4-20-j4ptp9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/dc12fe55-3a9b-4a1f-8c88-2777eae6961c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/dc12fe55-3a9b-4a1f-8c88-2777eae6961c\",\n      \"name\": \"dc12fe55-3a9b-4a1f-8c88-2777eae6961c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:05:49.652398803Z\",\n      \"endTime\": \"2026-07-20T17:24:58.42709892Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-6dlm44\" with version 4.20.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9c180\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-6dlm44: failed waiting for cluster=\"cluster-zstream-4-20-6dlm44\" in resourcegroup=\"rg-zstream-upgrade-4-20-j4ptp9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/dc12fe55-3a9b-4a1f-8c88-2777eae6961c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/dc12fe55-3a9b-4a1f-8c88-2777eae6961c\",\n      \"name\": \"dc12fe55-3a9b-4a1f-8c88-2777eae6961c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:05:49.652398803Z\",\n      \"endTime\": \"2026-07-20T17:24:58.42709892Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1ef723383d9abb2c4d2357a04e7d636d002488285b0b1c888c97c3e2456a6eb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"9dfae6e5978c1295fffad1fa285c0f56737b3e5ba630e35f78fc7c230070cc36","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"053c60, 0xc0004e9b00}, {{0xc000f8eec0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc00190f730 sp=0xc00190f618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc00190f888 sp=0xc00190f730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f2434053c60?, 0xc0004e9b00?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc00190f8b0 sp=0xc00190f888 pc=0x6cbcad3\nruntime.call16(0xc0004bf380, 0xc0006f95c0, 0x0, 0x0, 0x0, 0x10, 0xc00190fb08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc00190f8d0 sp=0xc00190f8b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0004e9b00?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc00190f910 sp=0xc00190f8d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0x870dbd9, 0x4}, {0xc000c24f60, 0x1, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc00190fe70 sp=0xc00190f910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0xc000c24f60?, 0xc000bc26e0?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc00190fee8 sp=0xc00190fe70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0004e9b00?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc00190ff88 sp=0xc00190fee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc00190ffe0 sp=0xc00190ff88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00190ffe8 sp=0xc00190ffe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 120\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 377 gp=0xc0016fa1c0 m=nil [select]:\nruntime.gopark(0xc000d5d900?, 0x2?, 0x0?, 0xbe?, 0xc000d5d8f4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000d5d780 sp=0xc000d5d760 pc=0x2f2350e\nruntime.selectgo(0xc000d5d900, 0xc000d5d8f0, 0xc0015dec80?, 0x0, 0x36f17dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000d5d8c0 sp=0xc000d5d780 pc=0x2efe2f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x932c9d8, 0xc00071cbd0}, 0x8775b21?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc000d5d930 sp=0xc000d5d8c0 pc=0x36f0f36\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x9377d20, {0x932c9d8, 0xc00071cbd0}, 0x11)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc000d5dc30 sp=0xc000d5d930 pc=0x3dfc6f2\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteResourceGroup({0x7f2434053c60, 0xc0004e9b00}, 0xc00154a960, 0xc000cc2de0, {0xc000c6ce10, 0x11}, 0x0, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/resourcegroup_helper.go:172 +0x6aa fp=0xc000d5dd90 sp=0xc000d5dc30 pc=0x6ca92ca\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00}, {0xc000c6ce10, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:568 +0x8a5 fp=0xc000d5df08 sp=0xc000d5dd90 pc=0x6c9fd05\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f2434053c60?, 0xc0004e9b00?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc000d5dfc0 sp=0xc000d5df08 pc=0x6c9d791\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000d5dfe0 sp=0xc000d5dfc0 pc=0x6c9d608\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000d5dfe8 sp=0xc000d5dfe0 pc=0x2f2bcc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 420\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\nrax    0xca\nrbx    0x0\nrcx    0x2f2dae3\nrdx    0x0\nrdi    0xc9599f0\nrsi    0x80\nrbp    0x7ffc3cc57520\nrsp    0x7ffc3cc574d8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x7\nr13    0x1\nr14    0xc957640\nr15    0x7\nrip    0x2f2dae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n053c60, 0xc0004e9b00}, {{0xc000f8eec0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc00190f730 sp=0xc00190f618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc00190f888 sp=0xc00190f730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f2434053c60?, 0xc0004e9b00?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc00190f8b0 sp=0xc00190f888 pc=0x6cbcad3\nruntime.call16(0xc0004bf380, 0xc0006f95c0, 0x0, 0x0, 0x0, 0x10, 0xc00190fb08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc00190f8d0 sp=0xc00190f8b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0004e9b00?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc00190f910 sp=0xc00190f8d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0x870dbd9, 0x4}, {0xc000c24f60, 0x1, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc00190fe70 sp=0xc00190f910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc0006f95c0?, 0xc000d70d20?}, {0xc000c24f60?, 0xc000bc26e0?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc00190fee8 sp=0xc00190fe70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0004e9b00?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc00190ff88 sp=0xc00190fee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc00190ffe0 sp=0xc00190ff88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc00190ffe8 sp=0xc00190ffe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 120\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 377 gp=0xc0016fa1c0 m=nil [select]:\nruntime.gopark(0xc000d5d900?, 0x2?, 0x0?, 0xbe?, 0xc000d5d8f4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000d5d780 sp=0xc000d5d760 pc=0x2f2350e\nruntime.selectgo(0xc000d5d900, 0xc000d5d8f0, 0xc0015dec80?, 0x0, 0x36f17dc?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000d5d8c0 sp=0xc000d5d780 pc=0x2efe2f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x932c9d8, 0xc00071cbd0}, 0x8775b21?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36 +0x76 fp=0xc000d5d930 sp=0xc000d5d8c0 pc=0x36f0f36\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x9377d20, {0x932c9d8, 0xc00071cbd0}, 0x11)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284 +0x732 fp=0xc000d5dc30 sp=0xc000d5d930 pc=0x3dfc6f2\ngithub.com/Azure/ARO-HCP/test/util/framework.DeleteResourceGroup({0x7f2434053c60, 0xc0004e9b00}, 0xc00154a960, 0xc000cc2de0, {0xc000c6ce10, 0x11}, 0x0, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/resourcegroup_helper.go:172 +0x6aa fp=0xc000d5dd90 sp=0xc000d5dc30 pc=0x6ca92ca\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).cleanupResourceGroup(0xc000bc26e0, {0x7f2434053c60, 0xc0004e9b00}, {0xc000c6ce10, 0x11}, 0x34630b8a000)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:568 +0x8a5 fp=0xc000d5df08 sp=0xc000d5dd90 pc=0x6c9fd05\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.func1({0x7f2434053c60?, 0xc0004e9b00?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:360 +0x151 fp=0xc000d5dfc0 sp=0xc000d5df08 pc=0x6c9d791\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups.gowrap1()\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:369 +0x28 fp=0xc000d5dfe0 sp=0xc000d5dfc0 pc=0x6c9d608\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000d5dfe8 sp=0xc000d5dfe0 pc=0x2f2bcc1\ncreated by github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups in goroutine 420\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:353 +0x98\n\nrax    0xca\nrbx    0x0\nrcx    0x2f2dae3\nrdx    0x0\nrdi    0xc9599f0\nrsi    0x80\nrbp    0x7ffc3cc57520\nrsp    0x7ffc3cc574d8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x7\nr13    0x1\nr14    0xc957640\nr15    0x7\nrip    0x2f2dae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}},{"row_id":"2658c4fe0cd8f4bd3881e56dab8ad1d77823ea0b4c8bdca5090efe881f091b95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"9f1a9e027c196576a3560509efe6090b4073d1d3f8fe6efe5c81506bd99aa7cb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-hjh4kk/cluster-candidate-4-21-65t9r9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00092c340\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-65t9r9: failed waiting for cluster=\"cluster-candidate-4-21-65t9r9\" in resourcegroup=\"rg-candidate-4-21-hjh4kk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3d5c1eb6-ed8a-4a20-b4e0-f916d47db593\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3d5c1eb6-ed8a-4a20-b4e0-f916d47db593\",\n      \"name\": \"3d5c1eb6-ed8a-4a20-b4e0-f916d47db593\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:04:00.634426013Z\",\n      \"endTime\": \"2026-07-20T17:23:18.533587615Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-hjh4kk/cluster-candidate-4-21-65t9r9 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00092c340\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-65t9r9: failed waiting for cluster=\"cluster-candidate-4-21-65t9r9\" in resourcegroup=\"rg-candidate-4-21-hjh4kk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3d5c1eb6-ed8a-4a20-b4e0-f916d47db593\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/3d5c1eb6-ed8a-4a20-b4e0-f916d47db593\",\n      \"name\": \"3d5c1eb6-ed8a-4a20-b4e0-f916d47db593\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:04:00.634426013Z\",\n      \"endTime\": \"2026-07-20T17:23:18.533587615Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"268600e25029f3e7e513b26189d632d6c366e071d45d91c05a153a56b16e4e5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"63e65a37b37f6351eb6b8822e95a229ef45f8da6b16599a8cc09a5bf14c22b6c","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9cea0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-wrhqrn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7c043db6-4547-4dff-8663-cbbb45a7f09b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7c043db6-4547-4dff-8663-cbbb45a7f09b\",\n      \"name\": \"7c043db6-4547-4dff-8663-cbbb45a7f09b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:17:54.986152146Z\",\n      \"endTime\": \"2026-07-20T16:36:57.261502092Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9cea0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-wrhqrn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7c043db6-4547-4dff-8663-cbbb45a7f09b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7c043db6-4547-4dff-8663-cbbb45a7f09b\",\n      \"name\": \"7c043db6-4547-4dff-8663-cbbb45a7f09b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:17:54.986152146Z\",\n      \"endTime\": \"2026-07-20T16:36:57.261502092Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2f714d2e6a241b0a601c47a43bd6954934323da770fa5a6d3f643fa50cfb0816","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"48e8d90cc93d877691af172e6ec9ce1cbf409e9c0749017c9ad86aa87c4eea0c","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:312]: failed to create HCP cluster np-noedge-bqqkqf\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e66620\u003e: \n    failed to create HCP cluster np-noedge-bqqkqf: failed waiting for cluster=\"np-noedge-bqqkqf\" in resourcegroup=\"rg-np-noedge-bqqkqf-h9dwjk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/537156bb-ec75-4dfc-9eaa-2e613bfeccc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/537156bb-ec75-4dfc-9eaa-2e613bfeccc5\",\n      \"name\": \"537156bb-ec75-4dfc-9eaa-2e613bfeccc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:23.596727955Z\",\n      \"endTime\": \"2026-07-20T18:10:39.449977043Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:312]: failed to create HCP cluster np-noedge-bqqkqf\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e66620\u003e: \n    failed to create HCP cluster np-noedge-bqqkqf: failed waiting for cluster=\"np-noedge-bqqkqf\" in resourcegroup=\"rg-np-noedge-bqqkqf-h9dwjk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/537156bb-ec75-4dfc-9eaa-2e613bfeccc5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/537156bb-ec75-4dfc-9eaa-2e613bfeccc5\",\n      \"name\": \"537156bb-ec75-4dfc-9eaa-2e613bfeccc5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:23.596727955Z\",\n      \"endTime\": \"2026-07-20T18:10:39.449977043Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"333ff93dde9e0021d1df4bc39c97d105fd40edb071e3ebeec1086e634b0afc29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"c2b3e56093fa347ac44a6564dc83b18e1731c958bd905686c717ec056b725ce7","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-pkpwnj with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00069a5c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-pkpwnj: failed waiting for cluster=\"np-version-upgrade-cluster-pkpwnj\" in resourcegroup=\"rg-np-version-upgrade-pkpwnj-k52f24\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a5eafffd-12ed-47f9-9d60-036543f6b686\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a5eafffd-12ed-47f9-9d60-036543f6b686\",\n      \"name\": \"a5eafffd-12ed-47f9-9d60-036543f6b686\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:00.884238157Z\",\n      \"endTime\": \"2026-07-20T18:10:19.645069551Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-pkpwnj with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00069a5c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-pkpwnj: failed waiting for cluster=\"np-version-upgrade-cluster-pkpwnj\" in resourcegroup=\"rg-np-version-upgrade-pkpwnj-k52f24\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a5eafffd-12ed-47f9-9d60-036543f6b686\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a5eafffd-12ed-47f9-9d60-036543f6b686\",\n      \"name\": \"a5eafffd-12ed-47f9-9d60-036543f6b686\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:00.884238157Z\",\n      \"endTime\": \"2026-07-20T18:10:19.645069551Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3423b4fe0f9153330226a8cb01bf5bc5a1d60f09a5beed790a5823d8fc00fd9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"e63bfecd2e80cd34d843b393871c75a9db632999606092b6f2b7879d05c4a6ce","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1a540\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-sqpj6p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6d995035-f3d7-41a6-a77e-fa47789e448e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6d995035-f3d7-41a6-a77e-fa47789e448e\",\n      \"name\": \"6d995035-f3d7-41a6-a77e-fa47789e448e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:07.28543095Z\",\n      \"endTime\": \"2026-07-20T16:37:17.106236321Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f1a540\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-sqpj6p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6d995035-f3d7-41a6-a77e-fa47789e448e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6d995035-f3d7-41a6-a77e-fa47789e448e\",\n      \"name\": \"6d995035-f3d7-41a6-a77e-fa47789e448e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:07.28543095Z\",\n      \"endTime\": \"2026-07-20T16:37:17.106236321Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3f1c25b5df7679190b8fe73f7dddc3041b85925b58e798b61a4975a54dce712c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"b889d85b96f7da35ce7694e811281973ebed2db00ab6b5be37bf7f7026eea10a","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c50f00\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-wb96pq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b074b0ac-92c8-47eb-a70e-6beb167049ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b074b0ac-92c8-47eb-a70e-6beb167049ae\",\n      \"name\": \"b074b0ac-92c8-47eb-a70e-6beb167049ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:08:03.365199196Z\",\n      \"endTime\": \"2026-07-20T17:27:18.710801787Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c50f00\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-wb96pq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b074b0ac-92c8-47eb-a70e-6beb167049ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b074b0ac-92c8-47eb-a70e-6beb167049ae\",\n      \"name\": \"b074b0ac-92c8-47eb-a70e-6beb167049ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:08:03.365199196Z\",\n      \"endTime\": \"2026-07-20T17:27:18.710801787Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"458fff335db1ea0ada5ea0a93bac4c8831e0b72d9291644c220c20d898ac71cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"efdc91261b3dc5be81a96c6a9bdf759132711c9bbb25b59724e796987c7c1da9","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dac6a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-jbhkmb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0af36a75-ce36-4b0c-96da-437af5e4a9cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0af36a75-ce36-4b0c-96da-437af5e4a9cc\",\n      \"name\": \"0af36a75-ce36-4b0c-96da-437af5e4a9cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:00.987915077Z\",\n      \"endTime\": \"2026-07-20T16:37:16.685766584Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dac6a0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-jbhkmb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0af36a75-ce36-4b0c-96da-437af5e4a9cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0af36a75-ce36-4b0c-96da-437af5e4a9cc\",\n      \"name\": \"0af36a75-ce36-4b0c-96da-437af5e4a9cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:00.987915077Z\",\n      \"endTime\": \"2026-07-20T16:37:16.685766584Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4e52567a8b9bbb134d490a396af6ce7b5eddab421c5ef4402b3cb135691a843b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"450380f4d7d6b972a7ee9e69bef0befd4fb47db16ae4787f70b470f4ead457f8","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000952d80\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-q8znls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/87ceb102-b0ec-4c82-980b-84d79a51deef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/87ceb102-b0ec-4c82-980b-84d79a51deef\",\n      \"name\": \"87ceb102-b0ec-4c82-980b-84d79a51deef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:05:40.800964096Z\",\n      \"endTime\": \"2026-07-20T17:24:48.200930424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000952d80\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-q8znls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/87ceb102-b0ec-4c82-980b-84d79a51deef\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/87ceb102-b0ec-4c82-980b-84d79a51deef\",\n      \"name\": \"87ceb102-b0ec-4c82-980b-84d79a51deef\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:05:40.800964096Z\",\n      \"endTime\": \"2026-07-20T17:24:48.200930424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"559715e7d66095c989d3ae3aa7835381e87ca5a11edb8b8ef40c020e24d57e37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"3e980bc9c2c2563cced522728967775c7c3b9bd16bf2013aa99e098a1290dab4","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000110200\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-2qqmns\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/20db9755-b153-4bb2-8706-0c0a4c914345\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/20db9755-b153-4bb2-8706-0c0a4c914345\",\n      \"name\": \"20db9755-b153-4bb2-8706-0c0a4c914345\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:06:08.019702048Z\",\n      \"endTime\": \"2026-07-20T17:25:18.208249507Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000110200\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-2qqmns\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/20db9755-b153-4bb2-8706-0c0a4c914345\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/20db9755-b153-4bb2-8706-0c0a4c914345\",\n      \"name\": \"20db9755-b153-4bb2-8706-0c0a4c914345\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:06:08.019702048Z\",\n      \"endTime\": \"2026-07-20T17:25:18.208249507Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5a7dde5ef9af13b69d2384fb1a3c541f4719b64eba492bbf43e30a0188eff0b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"c9ef03a3c161cff519a0cc77dd1fa8a976c7e422f3986b110e834eef903df89a","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9c560\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-dx4wm6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ef24c0d4-f3ad-4935-8711-e4bc4f11c7dc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ef24c0d4-f3ad-4935-8711-e4bc4f11c7dc\",\n      \"name\": \"ef24c0d4-f3ad-4935-8711-e4bc4f11c7dc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:06.065322164Z\",\n      \"endTime\": \"2026-07-20T16:37:16.698553789Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e9c560\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-dx4wm6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ef24c0d4-f3ad-4935-8711-e4bc4f11c7dc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ef24c0d4-f3ad-4935-8711-e4bc4f11c7dc\",\n      \"name\": \"ef24c0d4-f3ad-4935-8711-e4bc4f11c7dc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:06.065322164Z\",\n      \"endTime\": \"2026-07-20T16:37:16.698553789Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5a9ca30b222046a2a93569d6e29aa99edeacc61c5ed4b7366cb9234556ca15d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"3cee5acabea09afd20fb8c308fc2e28b46dce646f25113002c6b57a3edc86e4f","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011eaba0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6z844q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/471255a0-9ea3-4828-a373-c34d03625965\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/471255a0-9ea3-4828-a373-c34d03625965\",\n      \"name\": \"471255a0-9ea3-4828-a373-c34d03625965\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:07:06.252833555Z\",\n      \"endTime\": \"2026-07-20T17:26:18.713895489Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011eaba0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-6z844q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/471255a0-9ea3-4828-a373-c34d03625965\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/471255a0-9ea3-4828-a373-c34d03625965\",\n      \"name\": \"471255a0-9ea3-4828-a373-c34d03625965\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:07:06.252833555Z\",\n      \"endTime\": \"2026-07-20T17:26:18.713895489Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5c0d21f2735db6b6aef9356c0554c43fa372ba414050243b1c1d13ee7752197a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"f1229bb403839cad4628996a6410a9af531a5da6f8e4d9788b841ca3b4a79600","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create HCP cluster np-skip-minor-9srmsm at version 4.22.6\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0a460\u003e: \n    failed to create HCP cluster np-skip-minor-9srmsm: failed waiting for cluster=\"np-skip-minor-9srmsm\" in resourcegroup=\"rg-np-skip-minor-9srmsm-w59bjd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/358f9e05-fb59-4073-962d-6950d83db1e2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/358f9e05-fb59-4073-962d-6950d83db1e2\",\n      \"name\": \"358f9e05-fb59-4073-962d-6950d83db1e2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:40.13869337Z\",\n      \"endTime\": \"2026-07-20T18:10:49.418079989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create HCP cluster np-skip-minor-9srmsm at version 4.22.6\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0a460\u003e: \n    failed to create HCP cluster np-skip-minor-9srmsm: failed waiting for cluster=\"np-skip-minor-9srmsm\" in resourcegroup=\"rg-np-skip-minor-9srmsm-w59bjd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/358f9e05-fb59-4073-962d-6950d83db1e2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/358f9e05-fb59-4073-962d-6950d83db1e2\",\n      \"name\": \"358f9e05-fb59-4073-962d-6950d83db1e2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:40.13869337Z\",\n      \"endTime\": \"2026-07-20T18:10:49.418079989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5c48b8948893aaad4f07d211888dc3fa1a249b9280d3c07f5334fb8d1b22e3b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"9cdf4b0d450d675e7353c83c727667ff2ca957686fd721c48d61cbeba18b0401","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0a620\u003e: \n    failed to create HCP cluster ea-list4r2wmh: failed waiting for cluster=\"ea-list4r2wmh\" in resourcegroup=\"ea-list-shdd4s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7ffef2d7-4143-40c9-856d-4dc5bda3fbcb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7ffef2d7-4143-40c9-856d-4dc5bda3fbcb\",\n      \"name\": \"7ffef2d7-4143-40c9-856d-4dc5bda3fbcb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T18:36:08.338350325Z\",\n      \"endTime\": \"2026-07-20T18:55:21.966255683Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0a620\u003e: \n    failed to create HCP cluster ea-list4r2wmh: failed waiting for cluster=\"ea-list4r2wmh\" in resourcegroup=\"ea-list-shdd4s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7ffef2d7-4143-40c9-856d-4dc5bda3fbcb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7ffef2d7-4143-40c9-856d-4dc5bda3fbcb\",\n      \"name\": \"7ffef2d7-4143-40c9-856d-4dc5bda3fbcb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T18:36:08.338350325Z\",\n      \"endTime\": \"2026-07-20T18:55:21.966255683Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"64d55331ed24067718574edae070b8c73a45a3456b39420bb33392333cacd407","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"423e79c0bd7d14affff972c4fb9cbe5fe4860e6a955ae7ba8d21ec78bceb77fd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-20-v9qp6p/cluster-candidate-4-20-wmszpp should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8e7c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-wmszpp: failed waiting for cluster=\"cluster-candidate-4-20-wmszpp\" in resourcegroup=\"rg-candidate-4-20-v9qp6p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fedb83c8-2aff-49ac-bdf8-eac1959cf8cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fedb83c8-2aff-49ac-bdf8-eac1959cf8cc\",\n      \"name\": \"fedb83c8-2aff-49ac-bdf8-eac1959cf8cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:03:58.653081502Z\",\n      \"endTime\": \"2026-07-20T17:22:58.501527449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-20-v9qp6p/cluster-candidate-4-20-wmszpp should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8e7c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-wmszpp: failed waiting for cluster=\"cluster-candidate-4-20-wmszpp\" in resourcegroup=\"rg-candidate-4-20-v9qp6p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fedb83c8-2aff-49ac-bdf8-eac1959cf8cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fedb83c8-2aff-49ac-bdf8-eac1959cf8cc\",\n      \"name\": \"fedb83c8-2aff-49ac-bdf8-eac1959cf8cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:03:58.653081502Z\",\n      \"endTime\": \"2026-07-20T17:22:58.501527449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"658348804ba74fd80a78f29a85655ae1b039408a34cb0bd10d62c00cf4a6cb90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"e11b11903d7834d0633cec99c8e016a5437f7fb4b223d336b68e93d4889ded6d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-tm88nc/cluster-candidate-4-22-4hz5js should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000894600\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-4hz5js: failed waiting for cluster=\"cluster-candidate-4-22-4hz5js\" in resourcegroup=\"rg-candidate-4-22-tm88nc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/71bedfd7-d5d5-41f3-9984-7a70f8375e7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/71bedfd7-d5d5-41f3-9984-7a70f8375e7e\",\n      \"name\": \"71bedfd7-d5d5-41f3-9984-7a70f8375e7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:04:54.732277444Z\",\n      \"endTime\": \"2026-07-20T17:23:58.453899678Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-tm88nc/cluster-candidate-4-22-4hz5js should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000894600\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-4hz5js: failed waiting for cluster=\"cluster-candidate-4-22-4hz5js\" in resourcegroup=\"rg-candidate-4-22-tm88nc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/71bedfd7-d5d5-41f3-9984-7a70f8375e7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/71bedfd7-d5d5-41f3-9984-7a70f8375e7e\",\n      \"name\": \"71bedfd7-d5d5-41f3-9984-7a70f8375e7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:04:54.732277444Z\",\n      \"endTime\": \"2026-07-20T17:23:58.453899678Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6bdf4eca3b4c6e8875729c0145d0d61fcfd5759f69ede82b8a45511a2dd1c411","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"4ea27abf7b6d110a622804fbc493b5e0ccf63ffca1e5c4741ed58f14ea7b9019","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010083a0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hm2647\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b195ba0-db76-482d-82fa-7f8b278c5db4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b195ba0-db76-482d-82fa-7f8b278c5db4\",\n      \"name\": \"0b195ba0-db76-482d-82fa-7f8b278c5db4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:50:23.335917328Z\",\n      \"endTime\": \"2026-07-20T18:09:39.840081605Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010083a0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-hm2647\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b195ba0-db76-482d-82fa-7f8b278c5db4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0b195ba0-db76-482d-82fa-7f8b278c5db4\",\n      \"name\": \"0b195ba0-db76-482d-82fa-7f8b278c5db4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:50:23.335917328Z\",\n      \"endTime\": \"2026-07-20T18:09:39.840081605Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6c45f19637f934660be17e2a131fd80f4610c63bbe29d95b8013c08b40e0329e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"c3c54235f5bab1ed193af1a8bf1c53c0ebaaa00d911fbec180e364f4a51f51ff","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a404e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-779bpb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c42f1b6e-7cf0-4e79-9eee-8459407f4c5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c42f1b6e-7cf0-4e79-9eee-8459407f4c5a\",\n      \"name\": \"c42f1b6e-7cf0-4e79-9eee-8459407f4c5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:54:15.703797368Z\",\n      \"endTime\": \"2026-07-20T18:13:19.446706509Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a404e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-779bpb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c42f1b6e-7cf0-4e79-9eee-8459407f4c5a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c42f1b6e-7cf0-4e79-9eee-8459407f4c5a\",\n      \"name\": \"c42f1b6e-7cf0-4e79-9eee-8459407f4c5a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:54:15.703797368Z\",\n      \"endTime\": \"2026-07-20T18:13:19.446706509Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6d5991716d5769e0d43468d21657905de048ca3b6442d36b09cc770cbc54d7f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"51686efd47937541b795284eabc7b92c47a94bf2f18851f1522b758fa9d3de37","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c921e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-j64qhc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d7ef4fec-5edc-4a34-992d-faa523ad2853\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d7ef4fec-5edc-4a34-992d-faa523ad2853\",\n      \"name\": \"d7ef4fec-5edc-4a34-992d-faa523ad2853\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:53:07.63945845Z\",\n      \"endTime\": \"2026-07-20T18:12:19.434995424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c921e0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-j64qhc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d7ef4fec-5edc-4a34-992d-faa523ad2853\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d7ef4fec-5edc-4a34-992d-faa523ad2853\",\n      \"name\": \"d7ef4fec-5edc-4a34-992d-faa523ad2853\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:53:07.63945845Z\",\n      \"endTime\": \"2026-07-20T18:12:19.434995424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6fd282cbdbb485d8d92d98d51abdca282707725c87b2d672d2204e5aab4198ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"afb41fc4fca05aecf9be71e40e15240a89a9e0ddb93a288c8fe5c13030ff05e1","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009fa5e0\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-httjb4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7b2f907a-b2fe-4229-8633-6fc353a1f177\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7b2f907a-b2fe-4229-8633-6fc353a1f177\",\n      \"name\": \"7b2f907a-b2fe-4229-8633-6fc353a1f177\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:50:17.547241464Z\",\n      \"endTime\": \"2026-07-20T18:09:19.686173299Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009fa5e0\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-httjb4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7b2f907a-b2fe-4229-8633-6fc353a1f177\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7b2f907a-b2fe-4229-8633-6fc353a1f177\",\n      \"name\": \"7b2f907a-b2fe-4229-8633-6fc353a1f177\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:50:17.547241464Z\",\n      \"endTime\": \"2026-07-20T18:09:19.686173299Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7120d8bab12e3a47b60c445f8afbe23088444b40d822cc2d0bff49b05c7b7350","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"1ba3f6f8325991f02948a4154356510c1249ac6272a12bae716bd95bed6e76a3","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-gvcrtt with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e544c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-gvcrtt: failed waiting for cluster=\"np-version-upgrade-cluster-gvcrtt\" in resourcegroup=\"rg-np-version-upgrade-gvcrtt-xqwc5w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5a5a2833-0ca0-4b5d-b11a-4b90da619a8c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5a5a2833-0ca0-4b5d-b11a-4b90da619a8c\",\n      \"name\": \"5a5a2833-0ca0-4b5d-b11a-4b90da619a8c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:08.728865403Z\",\n      \"endTime\": \"2026-07-20T18:10:19.820799544Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-gvcrtt with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e544c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-gvcrtt: failed waiting for cluster=\"np-version-upgrade-cluster-gvcrtt\" in resourcegroup=\"rg-np-version-upgrade-gvcrtt-xqwc5w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5a5a2833-0ca0-4b5d-b11a-4b90da619a8c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5a5a2833-0ca0-4b5d-b11a-4b90da619a8c\",\n      \"name\": \"5a5a2833-0ca0-4b5d-b11a-4b90da619a8c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:08.728865403Z\",\n      \"endTime\": \"2026-07-20T18:10:19.820799544Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"81f3fc3167a80944fdfcdc49f087594f647f7f71ef70776b9f873a89d82463cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"3a3e1ebdf2af97f66ec9ef78ecb1afe1fb84673af192d4e99b9ebac1bd53db84","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014ec660\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-6kbj5v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8ad591a9-9e97-433f-b703-aec2e06c3851\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8ad591a9-9e97-433f-b703-aec2e06c3851\",\n      \"name\": \"8ad591a9-9e97-433f-b703-aec2e06c3851\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:17:55.004468099Z\",\n      \"endTime\": \"2026-07-20T16:36:57.134340873Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014ec660\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-6kbj5v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8ad591a9-9e97-433f-b703-aec2e06c3851\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8ad591a9-9e97-433f-b703-aec2e06c3851\",\n      \"name\": \"8ad591a9-9e97-433f-b703-aec2e06c3851\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:17:55.004468099Z\",\n      \"endTime\": \"2026-07-20T16:36:57.134340873Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8b01758357721812a0ceb2a84a3a16b4c2ebe2bb25575b2544edf7b920487d17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"56f87fb5a807646e254bd4c95e1d7baae5a9e2cc8f9b0f9e1c6f0454d4a97551","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016028d0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9zsl7m\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9zsl7m, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016028d0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-9zsl7m\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-9zsl7m, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-0-vxt256, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: failed waiting for cluster=\"cluster-nightly-5-0-vxt256\" in resourcegroup=\"rg-nightly-5-0-cpmsgk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-vxt256 in resource group rg-nightly-5-0-cpmsgk, error: context deadline exceeded"}},{"row_id":"8b88b6d9591ee9a64ced80c39dcd9bd939abdf7ef92c09868a347afbff9c7ebd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"ac3d836b0cc84b6d8e6cbd78dcc6f905c310e0ae1a869903ab9785d6f9f7c79d","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dee400\u003e: \n    failed to create HCP cluster np-downgrade-zlnpvz: failed waiting for cluster=\"np-downgrade-zlnpvz\" in resourcegroup=\"rg-np-downgrade-zlnpvz-474nfv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/72b18c43-9bb3-4395-893b-51c33b79054c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/72b18c43-9bb3-4395-893b-51c33b79054c\",\n      \"name\": \"72b18c43-9bb3-4395-893b-51c33b79054c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:45.683730612Z\",\n      \"endTime\": \"2026-07-20T18:10:49.792207288Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dee400\u003e: \n    failed to create HCP cluster np-downgrade-zlnpvz: failed waiting for cluster=\"np-downgrade-zlnpvz\" in resourcegroup=\"rg-np-downgrade-zlnpvz-474nfv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/72b18c43-9bb3-4395-893b-51c33b79054c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/72b18c43-9bb3-4395-893b-51c33b79054c\",\n      \"name\": \"72b18c43-9bb3-4395-893b-51c33b79054c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:45.683730612Z\",\n      \"endTime\": \"2026-07-20T18:10:49.792207288Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8ff9afbf7a5d19f4e47e942baba62899aa833bc02faafbff4c2a659752c9d1b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"c1d298712c5c3ea2c0294d603cda00c713c367daeedb531b3b9828faf8e5582b","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00060b060\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-89796h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6f2f4d55-a072-460d-890f-246667225587\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6f2f4d55-a072-460d-890f-246667225587\",\n      \"name\": \"6f2f4d55-a072-460d-890f-246667225587\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:07.556138017Z\",\n      \"endTime\": \"2026-07-20T16:37:17.037192315Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00060b060\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-89796h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6f2f4d55-a072-460d-890f-246667225587\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6f2f4d55-a072-460d-890f-246667225587\",\n      \"name\": \"6f2f4d55-a072-460d-890f-246667225587\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:18:07.556138017Z\",\n      \"endTime\": \"2026-07-20T16:37:17.037192315Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"931eda6e0ec9aebc2f4bed7480e038646ed51b1ded9086cd207e401f6e41068b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"ef690185b55d76436da7c8d08b241833303008c4ac0da413e4ac3bbff2970319","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000958680\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-zz6s2n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cee811fc-ac9c-4c6a-9336-c06a343317b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cee811fc-ac9c-4c6a-9336-c06a343317b5\",\n      \"name\": \"cee811fc-ac9c-4c6a-9336-c06a343317b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:53.535071338Z\",\n      \"endTime\": \"2026-07-20T18:11:00.258308483Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000958680\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-zz6s2n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cee811fc-ac9c-4c6a-9336-c06a343317b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cee811fc-ac9c-4c6a-9336-c06a343317b5\",\n      \"name\": \"cee811fc-ac9c-4c6a-9336-c06a343317b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:53.535071338Z\",\n      \"endTime\": \"2026-07-20T18:11:00.258308483Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9a54414c8f88fec2762ae9593623b02e2ec47d8795d57a8b89413bb13c727d63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"1eb11187eefe0b01a1fa6cc9a8e1ec084c4abe167cbe4877ca1d9800d8c201ef","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-m7pf22\" with version 4.22.4 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8a240\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-m7pf22: failed waiting for cluster=\"cluster-zstream-4-22-m7pf22\" in resourcegroup=\"rg-zstream-upgrade-4-22-5754g9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/134663f3-b85b-4f95-aaa5-036030778a08\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/134663f3-b85b-4f95-aaa5-036030778a08\",\n      \"name\": \"134663f3-b85b-4f95-aaa5-036030778a08\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:05:35.316811643Z\",\n      \"endTime\": \"2026-07-20T17:24:38.212528117Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-m7pf22\" with version 4.22.4 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8a240\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-m7pf22: failed waiting for cluster=\"cluster-zstream-4-22-m7pf22\" in resourcegroup=\"rg-zstream-upgrade-4-22-5754g9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/134663f3-b85b-4f95-aaa5-036030778a08\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/134663f3-b85b-4f95-aaa5-036030778a08\",\n      \"name\": \"134663f3-b85b-4f95-aaa5-036030778a08\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:05:35.316811643Z\",\n      \"endTime\": \"2026-07-20T17:24:38.212528117Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9d6ca583a4c4f3afa0e1b6c74104746067044cfca6751b8e632d419c285fb5b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"8a8d6f05b6dcccf74f063e9e884d589a943e2450c74ae8f06a0e16c3e05d42bb","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e14b80\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-9skv5p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/201959ef-a16a-47de-b332-8879a3c4907b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/201959ef-a16a-47de-b332-8879a3c4907b\",\n      \"name\": \"201959ef-a16a-47de-b332-8879a3c4907b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:49:47.640842257Z\",\n      \"endTime\": \"2026-07-20T18:08:49.417214158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e14b80\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-9skv5p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/201959ef-a16a-47de-b332-8879a3c4907b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/201959ef-a16a-47de-b332-8879a3c4907b\",\n      \"name\": \"201959ef-a16a-47de-b332-8879a3c4907b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:49:47.640842257Z\",\n      \"endTime\": \"2026-07-20T18:08:49.417214158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"acfd7869d01d966b7395e3e349bbf75a959f83c69e50df78e07d42a0ccd93256","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"275581ffbc1cf8d0c51e95c081ed302acf95313ca60b12f03dbeebfeed6f2c5e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-45h4f8\" with version 4.19.37 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d76b00\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-45h4f8: failed waiting for cluster=\"cluster-zstream-4-19-45h4f8\" in resourcegroup=\"rg-zstream-upgrade-4-19-dpg7px\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/98c43fdd-5cd7-4cc7-a362-b77fd854b20a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/98c43fdd-5cd7-4cc7-a362-b77fd854b20a\",\n      \"name\": \"98c43fdd-5cd7-4cc7-a362-b77fd854b20a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:04:53.582585323Z\",\n      \"endTime\": \"2026-07-20T17:23:58.453659574Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-45h4f8\" with version 4.19.37 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d76b00\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-45h4f8: failed waiting for cluster=\"cluster-zstream-4-19-45h4f8\" in resourcegroup=\"rg-zstream-upgrade-4-19-dpg7px\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/98c43fdd-5cd7-4cc7-a362-b77fd854b20a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/98c43fdd-5cd7-4cc7-a362-b77fd854b20a\",\n      \"name\": \"98c43fdd-5cd7-4cc7-a362-b77fd854b20a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:04:53.582585323Z\",\n      \"endTime\": \"2026-07-20T17:23:58.453659574Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b20e9158eb992c5adea1403f39bad217b8812b87455c1557ab7e99459e36c7ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"0829537e94cd357a37523e33879e0e7e22af64f406be9b39e166b0f3fa1db608","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c4ca20\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ksvhl5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8f09b243-ae51-46ba-959a-9bdddedd1d4d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8f09b243-ae51-46ba-959a-9bdddedd1d4d\",\n      \"name\": \"8f09b243-ae51-46ba-959a-9bdddedd1d4d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:06:30.075955701Z\",\n      \"endTime\": \"2026-07-20T17:25:48.195692162Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c4ca20\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ksvhl5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8f09b243-ae51-46ba-959a-9bdddedd1d4d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8f09b243-ae51-46ba-959a-9bdddedd1d4d\",\n      \"name\": \"8f09b243-ae51-46ba-959a-9bdddedd1d4d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:06:30.075955701Z\",\n      \"endTime\": \"2026-07-20T17:25:48.195692162Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b327b148aa0022e098e62d9bdec1c046d334c2775fb2348378bcfb7c6879f643","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"71677fd8d99a94c4e313dfaba10dd7a548ca9df1c53cef2d7ecdf60cc2bbcbe4","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:181]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-gxlkkp\" with install version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010e6160\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-gxlkkp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-gxlkkp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-f4jwjk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/197415b6-02bb-4107-b9cc-08c504e38f8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/197415b6-02bb-4107-b9cc-08c504e38f8d\",\n      \"name\": \"197415b6-02bb-4107-b9cc-08c504e38f8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:05:31.092655458Z\",\n      \"endTime\": \"2026-07-20T17:24:38.199264603Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:181]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-gxlkkp\" with install version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010e6160\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-gxlkkp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-gxlkkp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-f4jwjk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/197415b6-02bb-4107-b9cc-08c504e38f8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/197415b6-02bb-4107-b9cc-08c504e38f8d\",\n      \"name\": \"197415b6-02bb-4107-b9cc-08c504e38f8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:05:31.092655458Z\",\n      \"endTime\": \"2026-07-20T17:24:38.199264603Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b3c3f4e571a5f82412ea9e397ae770b2aa18b855ba88960cb610ad1d4ee18ff6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"58c983d115a51045c4dc82de9235833a1e9b75ec570b84d7e4598afe936ac62e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-21-6zmtbg\" with version 4.21.24 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b0900\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-6zmtbg: failed waiting for cluster=\"cluster-zstream-4-21-6zmtbg\" in resourcegroup=\"rg-zstream-upgrade-4-21-47m5v8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6f1495ef-2de3-43f9-b92e-f0627f8c87ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6f1495ef-2de3-43f9-b92e-f0627f8c87ba\",\n      \"name\": \"6f1495ef-2de3-43f9-b92e-f0627f8c87ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:04:43.154308183Z\",\n      \"endTime\": \"2026-07-20T17:23:58.31711227Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-21-6zmtbg\" with version 4.21.24 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b0900\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-6zmtbg: failed waiting for cluster=\"cluster-zstream-4-21-6zmtbg\" in resourcegroup=\"rg-zstream-upgrade-4-21-47m5v8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6f1495ef-2de3-43f9-b92e-f0627f8c87ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6f1495ef-2de3-43f9-b92e-f0627f8c87ba\",\n      \"name\": \"6f1495ef-2de3-43f9-b92e-f0627f8c87ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:04:43.154308183Z\",\n      \"endTime\": \"2026-07-20T17:23:58.31711227Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b5ff68cda13cf19639caaf3395bbf5ac84a10f3a3a67039d9f83286734cbfcdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"b6f4e9f1f6c7f0e0d12f6274441cf72f39009053ff85d2d81ef74cebb5deb612","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"3089fb7\nio.ReadAtLeast({0x92938e0, 0xc0014ab680}, {0xc0014b4204, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000af7d00 sp=0xc000af7cb8 pc=0x2f7e06e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0014b4204, 0x9, 0xc000000063?}, {0x92938e0?, 0xc0014ab680?})\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:250 +0x65 fp=0xc000af7d50 sp=0xc000af7d00 pc=0x35ee225\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:513 +0x6b fp=0xc000af7dd8 sp=0xc000af7d50 pc=0x35eedeb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:572 +0x18 fp=0xc000af7e18 sp=0xc000af7dd8 pc=0x35ef558\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000af7fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1878 +0xca fp=0xc000af7f60 sp=0xc000af7e18 pc=0x360154a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0016c0a80)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1760 +0x52 fp=0xc000af7fc8 sp=0xc000af7f60 pc=0x3600a72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0x25 fp=0xc000af7fe0 sp=0xc000af7fc8 pc=0x35fa1a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000af7fe8 sp=0xc000af7fe0 pc=0x2f2bcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 134\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0xda5\n\ngoroutine 316 gp=0xc000697dc0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0xc0016115b0?, 0x2eba874?, 0x20?, 0x99?, 0x200c001546540?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001611550 sp=0xc001611530 pc=0x2f2350e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc001558348, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc0016115b8 sp=0xc001611550 pc=0x2eff59d\nsync.runtime_SemacquireWaitGroup(0xc000d25920?, 0xc0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc0016115f0 sp=0xc0016115b8 pc=0x2f24c2e\nsync.(*WaitGroup).Wait(0xc001558340)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc001611618 sp=0xc0016115f0 pc=0x2f48405\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0}, {{0xc000ec09c0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc001611730 sp=0xc001611618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc001611888 sp=0xc001611730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f6c39063a60?, 0xc0015a32c0?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc0016118b0 sp=0xc001611888 pc=0x6cbcad3\nruntime.call16(0xc000e94300, 0xc000ae0410, 0x0, 0x0, 0x0, 0x10, 0xc001611b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc0016118d0 sp=0xc0016118b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0015a32c0?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc001611910 sp=0xc0016118d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc000ae0410?, 0xc000e8c780?}, {0x870dbd9, 0x4}, {0xc000879f60, 0x1, 0xc001476a00?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc001611e70 sp=0xc001611910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc000ae0410?, 0xc000879f68?}, {0xc000879f60?, 0xc000eba000?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc001611ee8 sp=0xc001611e70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0015a32c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc001611f88 sp=0xc001611ee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc001611fe0 sp=0xc001611f88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001611fe8 sp=0xc001611fe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 83\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xca\nrbx    0x0\nrcx    0x2f2dae3\nrdx    0x0\nrdi    0xc9599f0\nrsi    0x80\nrbp    0x7ffe78eb87a8\nrsp    0x7ffe78eb8760\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x4c\nr13    0x1\nr14    0xc957640\nr15    0x1\nrip    0x2f2dae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command\n3089fb7\nio.ReadAtLeast({0x92938e0, 0xc0014ab680}, {0xc0014b4204, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000af7d00 sp=0xc000af7cb8 pc=0x2f7e06e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0014b4204, 0x9, 0xc000000063?}, {0x92938e0?, 0xc0014ab680?})\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:250 +0x65 fp=0xc000af7d50 sp=0xc000af7d00 pc=0x35ee225\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:513 +0x6b fp=0xc000af7dd8 sp=0xc000af7d50 pc=0x35eedeb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0014b41c0)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/frame.go:572 +0x18 fp=0xc000af7e18 sp=0xc000af7dd8 pc=0x35ef558\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000af7fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1878 +0xca fp=0xc000af7f60 sp=0xc000af7e18 pc=0x360154a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0016c0a80)\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:1760 +0x52 fp=0xc000af7fc8 sp=0xc000af7f60 pc=0x3600a72\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0x25 fp=0xc000af7fe0 sp=0xc000af7fc8 pc=0x35fa1a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000af7fe8 sp=0xc000af7fe0 pc=0x2f2bcc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 134\n\t/go/pkg/mod/golang.org/x/net@v0.55.0/http2/transport.go:554 +0xda5\n\ngoroutine 316 gp=0xc000697dc0 m=nil [sync.WaitGroup.Wait]:\nruntime.gopark(0xc0016115b0?, 0x2eba874?, 0x20?, 0x99?, 0x200c001546540?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc001611550 sp=0xc001611530 pc=0x2f2350e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc001558348, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc0016115b8 sp=0xc001611550 pc=0x2eff59d\nsync.runtime_SemacquireWaitGroup(0xc000d25920?, 0xc0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc0016115f0 sp=0xc0016115b8 pc=0x2f24c2e\nsync.(*WaitGroup).Wait(0xc001558340)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc001611618 sp=0xc0016115f0 pc=0x2f48405\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0}, {{0xc000ec09c0, 0x1, 0x1}, 0x34630b8a000, {0x871a24d, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc001611730 sp=0xc001611618 pc=0x6c9d292\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000eba000, {0x7f6c39063a60, 0xc0015a32c0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc001611888 sp=0xc001611730 pc=0x6c9c9f0\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7f6c39063a60?, 0xc0015a32c0?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc0016118b0 sp=0xc001611888 pc=0x6cbcad3\nruntime.call16(0xc000e94300, 0xc000ae0410, 0x0, 0x0, 0x0, 0x10, 0xc001611b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc0016118d0 sp=0xc0016118b0 pc=0x2f2a223\nruntime.reflectcall(0x80e5200?, 0xc0015a32c0?, 0x2?, 0x8751e1d?, 0x0?, 0x12?, 0x7c9d5e0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc001611910 sp=0xc0016118d0 pc=0x2f32ab6\nreflect.Value.call({0x75465a0?, 0xc000ae0410?, 0xc000e8c780?}, {0x870dbd9, 0x4}, {0xc000879f60, 0x1, 0xc001476a00?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc001611e70 sp=0xc001611910 pc=0x2f65c66\nreflect.Value.Call({0x75465a0?, 0xc000ae0410?, 0xc000879f68?}, {0xc000879f60?, 0xc000eba000?, 0x932c968?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc001611ee8 sp=0xc001611e70 pc=0x2f64d59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x93472c8?, 0xc0015a32c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc001611f88 sp=0xc001611ee8 pc=0x3388c70\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc001611fe0 sp=0xc001611f88 pc=0x339b73b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001611fe8 sp=0xc001611fe0 pc=0x2f2bcc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 83\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xca\nrbx    0x0\nrcx    0x2f2dae3\nrdx    0x0\nrdi    0xc9599f0\nrsi    0x80\nrbp    0x7ffe78eb87a8\nrsp    0x7ffe78eb8760\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x4c\nr13    0x1\nr14    0xc957640\nr15    0x1\nrip    0x2f2dae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserialization Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","failure_pattern":"Deserialization Error: no output from command","search_query":"Deserialization Error: no output from command"}},{"row_id":"bd040bac37af1dd7399a6f9468f71e0f0923b1f13fd5b74d29d0cdf75d09636e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"4e01e47c19ed58d4cb8d2a8f24b3122e880837292a508157c286937fd0f35b84","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c0ae40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-lffq4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e551f69f-3ad8-4324-975f-6a196a46c1de\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e551f69f-3ad8-4324-975f-6a196a46c1de\",\n      \"name\": \"e551f69f-3ad8-4324-975f-6a196a46c1de\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:00.807585389Z\",\n      \"endTime\": \"2026-07-20T18:10:09.420000126Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c0ae40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-lffq4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e551f69f-3ad8-4324-975f-6a196a46c1de\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e551f69f-3ad8-4324-975f-6a196a46c1de\",\n      \"name\": \"e551f69f-3ad8-4324-975f-6a196a46c1de\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:51:00.807585389Z\",\n      \"endTime\": \"2026-07-20T18:10:09.420000126Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c19fa4620b91e82ded5a13be3b5ea343e00d2f6555422bc8881ef565995f1788","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"c740ce0cce9ebe2bdaa9cae606c203d15377f801e02d913a7b3128b5ab7a83e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"4adf1d1d4510d6acde5ea4f682631337eef5d8ce5834433a0b9b23e4af0dd41a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-54sh25/cluster-candidate-5-0-2qxdcj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a64f80\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-2qxdcj: failed waiting for cluster=\"cluster-candidate-5-0-2qxdcj\" in resourcegroup=\"rg-candidate-5-0-54sh25\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3fc52e2-6273-4c6b-b9fe-958a375e9be2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3fc52e2-6273-4c6b-b9fe-958a375e9be2\",\n      \"name\": \"e3fc52e2-6273-4c6b-b9fe-958a375e9be2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:04:41.800474139Z\",\n      \"endTime\": \"2026-07-20T17:23:58.315072737Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-54sh25/cluster-candidate-5-0-2qxdcj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a64f80\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-2qxdcj: failed waiting for cluster=\"cluster-candidate-5-0-2qxdcj\" in resourcegroup=\"rg-candidate-5-0-54sh25\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3fc52e2-6273-4c6b-b9fe-958a375e9be2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3fc52e2-6273-4c6b-b9fe-958a375e9be2\",\n      \"name\": \"e3fc52e2-6273-4c6b-b9fe-958a375e9be2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:04:41.800474139Z\",\n      \"endTime\": \"2026-07-20T17:23:58.315072737Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ee8c3d26e62f72f48eac5cc97f44b09b9017a840fc6b2daa320bdf987a5e8361","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"fe57b1f7e029db16afe6d13d5ca1bd16e0a7f7981e23844e85e0d5c49e7dc6e7","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-2v9hgq with version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc50c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-2v9hgq: failed waiting for cluster=\"np-version-upgrade-cluster-2v9hgq\" in resourcegroup=\"rg-np-version-upgrade-2v9hgq-mnksn4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90cd55f5-fa73-4d3c-a716-f849f4605a13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90cd55f5-fa73-4d3c-a716-f849f4605a13\",\n      \"name\": \"90cd55f5-fa73-4d3c-a716-f849f4605a13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:50:58.800649836Z\",\n      \"endTime\": \"2026-07-20T18:10:00.222985677Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-2v9hgq with version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cc50c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-2v9hgq: failed waiting for cluster=\"np-version-upgrade-cluster-2v9hgq\" in resourcegroup=\"rg-np-version-upgrade-2v9hgq-mnksn4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90cd55f5-fa73-4d3c-a716-f849f4605a13\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90cd55f5-fa73-4d3c-a716-f849f4605a13\",\n      \"name\": \"90cd55f5-fa73-4d3c-a716-f849f4605a13\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:50:58.800649836Z\",\n      \"endTime\": \"2026-07-20T18:10:00.222985677Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f3443124360d7a9ac7c617458fa0ecd2f7cbf3d1b79851ab993eecfa1898f224","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"9707201f4c3543174d8c2063e2963ddfc92c45cddea94c8422c38fda847d1c86","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c32b40\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-l6nk7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d3134b27-f3d1-47f1-9586-af4dc6c51520\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d3134b27-f3d1-47f1-9586-af4dc6c51520\",\n      \"name\": \"d3134b27-f3d1-47f1-9586-af4dc6c51520\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:52:34.467585628Z\",\n      \"endTime\": \"2026-07-20T18:11:39.44183724Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c32b40\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-l6nk7z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d3134b27-f3d1-47f1-9586-af4dc6c51520\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d3134b27-f3d1-47f1-9586-af4dc6c51520\",\n      \"name\": \"d3134b27-f3d1-47f1-9586-af4dc6c51520\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T17:52:34.467585628Z\",\n      \"endTime\": \"2026-07-20T18:11:39.44183724Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f4769cb06533e49c2518f7668de99ea489a6b88ca4e5a547cd806ffbee9daa1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079235454674145280","occurred_at":"2026-07-20T16:02:17Z","failed_at":"e2e","signature_id":"403b05be2c136111f81b99d81a31731e246e1d1701de8e6d380aceec11313d30","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:150]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ced480\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-pll2br\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8d2bc806-e523-4d6f-b758-feebf4bbe60f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8d2bc806-e523-4d6f-b758-feebf4bbe60f\",\n      \"name\": \"8d2bc806-e523-4d6f-b758-feebf4bbe60f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:19:24.843029182Z\",\n      \"endTime\": \"2026-07-20T16:38:26.889267813Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:150]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ced480\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-pll2br\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8d2bc806-e523-4d6f-b758-feebf4bbe60f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8d2bc806-e523-4d6f-b758-feebf4bbe60f\",\n      \"name\": \"8d2bc806-e523-4d6f-b758-feebf4bbe60f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T16:19:24.843029182Z\",\n      \"endTime\": \"2026-07-20T16:38:26.889267813Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":45,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","a1df4a102ffa44f97ddbf6779aa0d2a52a0f6f8f1f86a9bebf09b9cdf20d5052","f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b"],"matched_occurrences":45,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079180487246483456","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-20T12:23:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079179464062799872","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-20T12:19:48Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079179226484838400","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-20T12:18:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079177126338105344","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-20T12:10:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079124531833737216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-20T08:38:32Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079074941612855296","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-20T05:24:28Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6280f72fc6380c3234b0ada5b775abcefb8910b3ed5171bcc9ac954654ee8f05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079074941612855296","occurred_at":"2026-07-20T05:24:28Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079023705853267968","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-20T02:00:52Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"014486c484344c68ff443506d40f7c13cbe314ef7688fef7440bee46d4d6ac3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079023705853267968","occurred_at":"2026-07-20T02:00:52Z","failed_at":"e2e","signature_id":"0d4e6c626552d37165a629be9c672bc072d0e8affb1fa7528cee84809b6aefcf","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a9c570\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-5-0-zkvjxb\", caused by: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a9c570\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-5-0-zkvjxb\", caused by: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-5-0-zkvjxb\", caused by: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-5-0-bh8jcp\" in resourcegroup=\"rg-candidate-5-0-zkvjxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-5-0-bh8jcp in resource group rg-candidate-5-0-zkvjxb, error: context deadline exceeded"}},{"row_id":"992c95c712558d132c8126150558a324e3cdec62ee682abe30116d1afee6ca08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079023705853267968","occurred_at":"2026-07-20T02:00:52Z","failed_at":"e2e","signature_id":"c15750545600e820878efaa487132ca680dc610368be4906d55af70ae65160b6","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012881e0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8d968x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3888477a-bc90-496f-be5a-132321ab3a6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3888477a-bc90-496f-be5a-132321ab3a6f\",\n      \"name\": \"3888477a-bc90-496f-be5a-132321ab3a6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T02:16:49.779634198Z\",\n      \"endTime\": \"2026-07-20T02:35:49.961113313Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012881e0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-8d968x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3888477a-bc90-496f-be5a-132321ab3a6f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3888477a-bc90-496f-be5a-132321ab3a6f\",\n      \"name\": \"3888477a-bc90-496f-be5a-132321ab3a6f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-20T02:16:49.779634198Z\",\n      \"endTime\": \"2026-07-20T02:35:49.961113313Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}