{"meta":{"date":"2026-07-21","timezone":"UTC","generated_at":"2026-08-26T20:24:57Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":9,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":6,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2079703153996992512","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-21T23:00:45Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2fa1c818241e7a18136ee4d557a418e8ac0c964451de79dfaba65b8977b24927","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2079703153996992512","occurred_at":"2026-07-21T23:00:45Z","failed_at":"e2e","signature_id":"0755e5951a373ee7739f2da37242ca447fa6e529080b3ff77c5a94299cead45f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-21-r4hvsk6dxsls/cluster-nightly-4-21-r2qnp7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c7ec40\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-r2qnp7: failed waiting for cluster=\"cluster-nightly-4-21-r2qnp7\" in resourcegroup=\"rg-nightly-4-21-r4hvsk6dxsls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\",\n      \"name\": \"4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T23:27:28.08522946Z\",\n      \"endTime\": \"2026-07-21T23:46:45.381055827Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-21-r4hvsk6dxsls/cluster-nightly-4-21-r2qnp7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c7ec40\u003e: \n    failed to create HCP cluster cluster-nightly-4-21-r2qnp7: failed waiting for cluster=\"cluster-nightly-4-21-r2qnp7\" in resourcegroup=\"rg-nightly-4-21-r4hvsk6dxsls\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\",\n      \"name\": \"4c053363-1ed8-4fe0-a9a2-5b7ec0c5f42c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T23:27:28.08522946Z\",\n      \"endTime\": \"2026-07-21T23:46:45.381055827Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8dbf2190a015fe901a9542ae8cdc035aabbbb2641b8c72ab7ab2d9417e163371","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2079703153996992512","occurred_at":"2026-07-21T23:00:45Z","failed_at":"e2e","signature_id":"e1a1c883dcefa03cf51fb3c9f2516413ee3f4efc3a25f372165982215efe1c15","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-20-sq54mx2hh8bz/cluster-nightly-4-20-l4wzgj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7aa40\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-l4wzgj: failed waiting for cluster=\"cluster-nightly-4-20-l4wzgj\" in resourcegroup=\"rg-nightly-4-20-sq54mx2hh8bz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5630f2e-a302-48b2-b58a-4b41f36eb4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5630f2e-a302-48b2-b58a-4b41f36eb4af\",\n      \"name\": \"a5630f2e-a302-48b2-b58a-4b41f36eb4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T23:05:38.885352827Z\",\n      \"endTime\": \"2026-07-21T23:24:44.270636482Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-nightly-4-20-sq54mx2hh8bz/cluster-nightly-4-20-l4wzgj should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7aa40\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-l4wzgj: failed waiting for cluster=\"cluster-nightly-4-20-l4wzgj\" in resourcegroup=\"rg-nightly-4-20-sq54mx2hh8bz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5630f2e-a302-48b2-b58a-4b41f36eb4af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5630f2e-a302-48b2-b58a-4b41f36eb4af\",\n      \"name\": \"a5630f2e-a302-48b2-b58a-4b41f36eb4af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T23:05:38.885352827Z\",\n      \"endTime\": \"2026-07-21T23:24:44.270636482Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079659653058269184","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-21T20:07:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079607054049218560","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-21T16:38:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-21T16:34:33Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"0bfccd11a874872633590b26ea0042735d805a900654208130b536984336835f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","failed_at":"e2e","signature_id":"0b60a1c8582172801768ad634340a733b420bceb7a417943da0687ed9f68b890","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-jf26hw/cluster-candidate-4-21-sbclb8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed8640\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-sbclb8: failed waiting for cluster=\"cluster-candidate-4-21-sbclb8\" in resourcegroup=\"rg-candidate-4-21-jf26hw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f284f15a-0205-423a-903d-1598351aa6b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f284f15a-0205-423a-903d-1598351aa6b7\",\n      \"name\": \"f284f15a-0205-423a-903d-1598351aa6b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:23:06.106421624Z\",\n      \"endTime\": \"2026-07-21T17:42:20.33106929Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-jf26hw/cluster-candidate-4-21-sbclb8 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed8640\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-sbclb8: failed waiting for cluster=\"cluster-candidate-4-21-sbclb8\" in resourcegroup=\"rg-candidate-4-21-jf26hw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f284f15a-0205-423a-903d-1598351aa6b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/f284f15a-0205-423a-903d-1598351aa6b7\",\n      \"name\": \"f284f15a-0205-423a-903d-1598351aa6b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:23:06.106421624Z\",\n      \"endTime\": \"2026-07-21T17:42:20.33106929Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"13ada5785a23ea5f79aabfe1ab1fc0282358f5a7faaeea23ab828de5e2f92425","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","failed_at":"e2e","signature_id":"8762cfe09fbb635fdeb6d2aeb186ff27d5dcfb284fb7d8cd73b9f9066590a09b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-n6rg6v/cluster-candidate-4-22-zw26wz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b0c00\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zw26wz: failed waiting for cluster=\"cluster-candidate-4-22-zw26wz\" in resourcegroup=\"rg-candidate-4-22-n6rg6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"name\": \"6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:00.417709536Z\",\n      \"endTime\": \"2026-07-21T17:45:01.077714445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-n6rg6v/cluster-candidate-4-22-zw26wz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009b0c00\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-zw26wz: failed waiting for cluster=\"cluster-candidate-4-22-zw26wz\" in resourcegroup=\"rg-candidate-4-22-n6rg6v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"name\": \"6d83515e-bf6d-403d-8393-495c72f0b7bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:00.417709536Z\",\n      \"endTime\": \"2026-07-21T17:45:01.077714445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c59269c250bd09f866d89f5097669316613ce7d5e4a7268ed28a2ccc21e5134c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3e4e7d02ad07ffac41130e9d3545b00392d6cc56dd0fa426da9a454638e9a6de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","failed_at":"e2e","signature_id":"69b586e970c0cce85494b13fd30b9055a1bcead75f883e0e702343732b8f47e6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-8r5shn/cluster-candidate-5-0-wj576z should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc260\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-wj576z: failed waiting for cluster=\"cluster-candidate-5-0-wj576z\" in resourcegroup=\"rg-candidate-5-0-8r5shn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"name\": \"3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:18.759928525Z\",\n      \"endTime\": \"2026-07-21T17:45:20.361591697Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-8r5shn/cluster-candidate-5-0-wj576z should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc260\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-wj576z: failed waiting for cluster=\"cluster-candidate-5-0-wj576z\" in resourcegroup=\"rg-candidate-5-0-8r5shn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"name\": \"3119b6a0-b91c-4658-8b22-178014841c9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:26:18.759928525Z\",\n      \"endTime\": \"2026-07-21T17:45:20.361591697Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: ignition-server-proxy deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ae125442940bde90789cccefefed6da7645f30b5d4ff58e7803d4ddb14827ec","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"42a052039adddb45b70357de2e703aa28812c00a168fd70c962027c3e6432b2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","failed_at":"e2e","signature_id":"41a62befca0336e9168b3f44954bf812cebf78b2ccbce1b2a930c0c01f99c53f","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00070a340\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9dp9s2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3f50b88f-788f-4024-9b88-812abadedfb8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3f50b88f-788f-4024-9b88-812abadedfb8\",\n      \"name\": \"3f50b88f-788f-4024-9b88-812abadedfb8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T16:54:05.440426627Z\",\n      \"endTime\": \"2026-07-21T17:13:19.012396307Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00070a340\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9dp9s2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3f50b88f-788f-4024-9b88-812abadedfb8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/3f50b88f-788f-4024-9b88-812abadedfb8\",\n      \"name\": \"3f50b88f-788f-4024-9b88-812abadedfb8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T16:54:05.440426627Z\",\n      \"endTime\": \"2026-07-21T17:13:19.012396307Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5a9000ae81cf1c8c98aeebfdaae5231a756abaad7417057c6a3fd8c7b4397ef5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","failed_at":"e2e","signature_id":"98902a321d14a054647d3e70ac37d774f906fdf4e97287d1c3c47c88206d1e19","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00178a160\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pkmlp7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/71408aa6-0732-42c1-9db6-29aa39e60059\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/71408aa6-0732-42c1-9db6-29aa39e60059\",\n      \"name\": \"71408aa6-0732-42c1-9db6-29aa39e60059\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T18:36:34.284034007Z\",\n      \"endTime\": \"2026-07-21T18:55:42.590587309Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00178a160\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pkmlp7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/71408aa6-0732-42c1-9db6-29aa39e60059\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/71408aa6-0732-42c1-9db6-29aa39e60059\",\n      \"name\": \"71408aa6-0732-42c1-9db6-29aa39e60059\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T18:36:34.284034007Z\",\n      \"endTime\": \"2026-07-21T18:55:42.590587309Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6d64ac4a0c919a4e0b1f18b2a7179834e07dd9f14ba2664dff90382800774fa6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","failed_at":"e2e","signature_id":"9013e675989bd3312d95cfd426e5318859765876455d6850dc98ee4d503feff7","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"7cee021d15d4a79b29482024447dd79e08a46690fad32593afac57d6d7f34702","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","failed_at":"e2e","signature_id":"bf8595f1f1b6f60837a216d52bfebf5779dc79f95e44ffe1574537f650c5689a","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a4c40\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-x4w55x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/7f995e11-6428-4d99-8c59-1e28c2e27c61\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/7f995e11-6428-4d99-8c59-1e28c2e27c61\",\n      \"name\": \"7f995e11-6428-4d99-8c59-1e28c2e27c61\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T16:52:06.426170661Z\",\n      \"endTime\": \"2026-07-21T17:11:19.040883829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007a4c40\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-x4w55x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/7f995e11-6428-4d99-8c59-1e28c2e27c61\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/7f995e11-6428-4d99-8c59-1e28c2e27c61\",\n      \"name\": \"7f995e11-6428-4d99-8c59-1e28c2e27c61\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T16:52:06.426170661Z\",\n      \"endTime\": \"2026-07-21T17:11:19.040883829Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e1c6578d9187a2392194c96cbb0c44ac5d2bf189bb66fcc92a9ed21d92faf057","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","failed_at":"e2e","signature_id":"cf71ca211aad8a02c62dc8a01cf969bfd91e010e4142091f8852b71ff5136236","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007987e0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-fh69wt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1cf25ed7-a65e-4752-be8d-6fe753bc6445\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1cf25ed7-a65e-4752-be8d-6fe753bc6445\",\n      \"name\": \"1cf25ed7-a65e-4752-be8d-6fe753bc6445\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:28:25.317884371Z\",\n      \"endTime\": \"2026-07-21T17:47:30.309220354Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007987e0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-fh69wt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1cf25ed7-a65e-4752-be8d-6fe753bc6445\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1cf25ed7-a65e-4752-be8d-6fe753bc6445\",\n      \"name\": \"1cf25ed7-a65e-4752-be8d-6fe753bc6445\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:28:25.317884371Z\",\n      \"endTime\": \"2026-07-21T17:47:30.309220354Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fedc1b935bcd0815caaf58aac5ac10ba0bfb4cf90d8d12dc316a6011763cb86c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079605965539250176","occurred_at":"2026-07-21T16:34:33Z","failed_at":"e2e","signature_id":"7212f4228fcb02a7c61725079499e0e2db3beea340f52127f257bbd6a5a4fde9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-20-dmm2nh/cluster-candidate-4-20-7jgrjb should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8af40\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-7jgrjb: failed waiting for cluster=\"cluster-candidate-4-20-7jgrjb\" in resourcegroup=\"rg-candidate-4-20-dmm2nh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1c9d7733-c8f8-4c56-aa0e-54984164fe8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1c9d7733-c8f8-4c56-aa0e-54984164fe8d\",\n      \"name\": \"1c9d7733-c8f8-4c56-aa0e-54984164fe8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:23:15.812647924Z\",\n      \"endTime\": \"2026-07-21T17:42:20.448701569Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-20-dmm2nh/cluster-candidate-4-20-7jgrjb should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e8af40\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-7jgrjb: failed waiting for cluster=\"cluster-candidate-4-20-7jgrjb\" in resourcegroup=\"rg-candidate-4-20-dmm2nh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1c9d7733-c8f8-4c56-aa0e-54984164fe8d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1c9d7733-c8f8-4c56-aa0e-54984164fe8d\",\n      \"name\": \"1c9d7733-c8f8-4c56-aa0e-54984164fe8d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T17:23:15.812647924Z\",\n      \"endTime\": \"2026-07-21T17:42:20.448701569Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] \\u003cno_message\\u003e; [cosmosCluster] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] \u003cno_message\u003e; [cosmosCluster] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","6ae125442940bde90789cccefefed6da7645f30b5d4ff58e7803d4ddb14827ec","c59269c250bd09f866d89f5097669316613ce7d5e4a7268ed28a2ccc21e5134c","e3a619978526b3447b4b56b4a2f19b2ca1e6cea88b8c710ff9d003c0f82eb47c"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079604628114116608","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-21T16:29:15Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079555388260225024","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-21T13:13:35Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-21T05:13:16Z"},"failed_at":["e2e"],"failed_test_count":45,"occurrences":[{"row_id":"00e84ab375bb4c5b49e33d2dd54d92bc5228f97404fbfc62249508b7b567b3c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"cd2b4b061f6b7f5f091d02b70ed0abb0daa4b65b9a703f77ede69a77a1f5ae57","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8c560\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ddp9qz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8c8f0685-7b86-4162-939d-ab9e185a8dc7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8c8f0685-7b86-4162-939d-ab9e185a8dc7\",\n      \"name\": \"8c8f0685-7b86-4162-939d-ab9e185a8dc7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:03:55.053747162Z\",\n      \"endTime\": \"2026-07-21T07:23:11.802187363Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8c560\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ddp9qz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8c8f0685-7b86-4162-939d-ab9e185a8dc7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8c8f0685-7b86-4162-939d-ab9e185a8dc7\",\n      \"name\": \"8c8f0685-7b86-4162-939d-ab9e185a8dc7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:03:55.053747162Z\",\n      \"endTime\": \"2026-07-21T07:23:11.802187363Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"05f94b625eeb635a1621a7628f0bf2796bcde63eaca6d26a37523706e39c3e32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"982a3df29aed4e7b407b3f82c328895ce2e66a9c7782e98b621b68dd60e800d5","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0c280\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gqqfmr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c11efeab-07f5-4778-80ad-f3fe25983680\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c11efeab-07f5-4778-80ad-f3fe25983680\",\n      \"name\": \"c11efeab-07f5-4778-80ad-f3fe25983680\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:18:03.976507732Z\",\n      \"endTime\": \"2026-07-21T06:37:20.808141431Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0c280\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gqqfmr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c11efeab-07f5-4778-80ad-f3fe25983680\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c11efeab-07f5-4778-80ad-f3fe25983680\",\n      \"name\": \"c11efeab-07f5-4778-80ad-f3fe25983680\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:18:03.976507732Z\",\n      \"endTime\": \"2026-07-21T06:37:20.808141431Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"08b67fe1dbf9e2b03e5bee657d7bc8400d2988431844a4861a34834894a40ac8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"3ef7d2dff7948260c72e4925bb6ed3d13b88e6cf1d66dfb25a785191e349e1dd","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-7pc9qw\" with version 4.22.4 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000828760\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-7pc9qw: failed waiting for cluster=\"cluster-zstream-4-22-7pc9qw\" in resourcegroup=\"rg-zstream-upgrade-4-22-prbn8g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e18ace24-930e-4f28-aad9-d52ea02360e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e18ace24-930e-4f28-aad9-d52ea02360e5\",\n      \"name\": \"e18ace24-930e-4f28-aad9-d52ea02360e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:16:08.616075795Z\",\n      \"endTime\": \"2026-07-21T06:35:10.987367432Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-22-7pc9qw\" with version 4.22.4 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000828760\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-7pc9qw: failed waiting for cluster=\"cluster-zstream-4-22-7pc9qw\" in resourcegroup=\"rg-zstream-upgrade-4-22-prbn8g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e18ace24-930e-4f28-aad9-d52ea02360e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e18ace24-930e-4f28-aad9-d52ea02360e5\",\n      \"name\": \"e18ace24-930e-4f28-aad9-d52ea02360e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:16:08.616075795Z\",\n      \"endTime\": \"2026-07-21T06:35:10.987367432Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0ba8346401bb2eb011475bb938f4362700b6e6742139684f339a523b33fd82f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"36eff7f126e8afb24c07e1a4dc4d242b4f6a538fd9427595143b9b10c3371cd3","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-84j7nm/cluster-candidate-5-0-kk2n48 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012144c0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-kk2n48: failed waiting for cluster=\"cluster-candidate-5-0-kk2n48\" in resourcegroup=\"rg-candidate-5-0-84j7nm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a86900c3-23a2-4009-aa6d-e8749c0f5d57\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a86900c3-23a2-4009-aa6d-e8749c0f5d57\",\n      \"name\": \"a86900c3-23a2-4009-aa6d-e8749c0f5d57\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:16:19.820038474Z\",\n      \"endTime\": \"2026-07-21T06:35:20.800149877Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-84j7nm/cluster-candidate-5-0-kk2n48 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012144c0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-kk2n48: failed waiting for cluster=\"cluster-candidate-5-0-kk2n48\" in resourcegroup=\"rg-candidate-5-0-84j7nm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a86900c3-23a2-4009-aa6d-e8749c0f5d57\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a86900c3-23a2-4009-aa6d-e8749c0f5d57\",\n      \"name\": \"a86900c3-23a2-4009-aa6d-e8749c0f5d57\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:16:19.820038474Z\",\n      \"endTime\": \"2026-07-21T06:35:20.800149877Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"11063ddccd01b11b4e4c1030ed473e9b9cb64fa4cd1777a9f41254cbafc4c140","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"6476a8257036dba2ef91b5f15019cbdd5701e7b188929c2fee04923ee80488ad","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e1bb60\u003e: \n    failed to create HCP cluster np-downgrade-brn7kf: failed waiting for cluster=\"np-downgrade-brn7kf\" in resourcegroup=\"rg-np-downgrade-brn7kf-rztswt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18782f52-dd7a-4d81-bbc0-4a6e63c7699b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18782f52-dd7a-4d81-bbc0-4a6e63c7699b\",\n      \"name\": \"18782f52-dd7a-4d81-bbc0-4a6e63c7699b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:36.184821317Z\",\n      \"endTime\": \"2026-07-21T07:21:41.788804976Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e1bb60\u003e: \n    failed to create HCP cluster np-downgrade-brn7kf: failed waiting for cluster=\"np-downgrade-brn7kf\" in resourcegroup=\"rg-np-downgrade-brn7kf-rztswt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18782f52-dd7a-4d81-bbc0-4a6e63c7699b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/18782f52-dd7a-4d81-bbc0-4a6e63c7699b\",\n      \"name\": \"18782f52-dd7a-4d81-bbc0-4a6e63c7699b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:36.184821317Z\",\n      \"endTime\": \"2026-07-21T07:21:41.788804976Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"134a0d45d2443244a8cda4c653d9961070e68d1f206b19f94333bd6289c8ecb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"79cb067dd7fce633fcf9600b882f8e03becb10a7fef0f4052b0f878f87663aa1","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b4b180\u003e: \n    failed to create HCP cluster np-dg-minor-x4mfkl: failed waiting for cluster=\"np-dg-minor-x4mfkl\" in resourcegroup=\"rg-np-dg-minor-x4mfkl-rbrh6t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/43644fe3-0f7e-4d4f-9583-c978150e3256\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/43644fe3-0f7e-4d4f-9583-c978150e3256\",\n      \"name\": \"43644fe3-0f7e-4d4f-9583-c978150e3256\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:44.098862626Z\",\n      \"endTime\": \"2026-07-21T07:21:51.795285874Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b4b180\u003e: \n    failed to create HCP cluster np-dg-minor-x4mfkl: failed waiting for cluster=\"np-dg-minor-x4mfkl\" in resourcegroup=\"rg-np-dg-minor-x4mfkl-rbrh6t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/43644fe3-0f7e-4d4f-9583-c978150e3256\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/43644fe3-0f7e-4d4f-9583-c978150e3256\",\n      \"name\": \"43644fe3-0f7e-4d4f-9583-c978150e3256\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:44.098862626Z\",\n      \"endTime\": \"2026-07-21T07:21:51.795285874Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1b6917b11e150bfd7e318e9b4cec8c46cf991eefe550236ac3872d44d573ac66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"0012df4f80a07c3b6a174e45ee4b577b91c683510b3ea91652ed742beb80e3a2","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9e360\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-htsfbf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"name\": \"c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:23.87500457Z\",\n      \"endTime\": \"2026-07-21T06:34:30.805495615Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f9e360\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-htsfbf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"name\": \"c99612e5-ef65-4612-87de-9bd468c43288\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:23.87500457Z\",\n      \"endTime\": \"2026-07-21T06:34:30.805495615Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b9e6c893f9d6b9c0dbf8393eaa7058a3e81a0f3edd443a11692869aa0138917","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"62e57ebc3b6e09155e29b65586f7eaa5453e4f355488112ed6ef3084f651edb2","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a4a1c0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-zdfkbp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/78ed358e-2c54-44ba-b9aa-361577bb2354\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/78ed358e-2c54-44ba-b9aa-361577bb2354\",\n      \"name\": \"78ed358e-2c54-44ba-b9aa-361577bb2354\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:28:54.840401134Z\",\n      \"endTime\": \"2026-07-21T05:47:58.658080215Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a4a1c0\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-zdfkbp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/78ed358e-2c54-44ba-b9aa-361577bb2354\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/78ed358e-2c54-44ba-b9aa-361577bb2354\",\n      \"name\": \"78ed358e-2c54-44ba-b9aa-361577bb2354\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:28:54.840401134Z\",\n      \"endTime\": \"2026-07-21T05:47:58.658080215Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3e94691576cef2b054996953d98f05bbaee533685ae0138e7a85a1bf5cfcf871","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"3945951f7e05773ea2b561ea7f6bdeed78c60d3a5f4fe590aa448bbbd2e9515c","test_name":"FIPS Mode Support with ExperimentalReleaseFeatures AFEC registered should create an HCP cluster with FIPS mode enabled via experimental tag","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f04f00\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-z4f9jd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c1bb3da1-1029-43c0-9c00-a1c0774a9343\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c1bb3da1-1029-43c0-9c00-a1c0774a9343\",\n      \"name\": \"c1bb3da1-1029-43c0-9c00-a1c0774a9343\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:13.461257101Z\",\n      \"endTime\": \"2026-07-21T06:34:20.998111005Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:84]: failed to create HCP cluster \"fips-enabled-cluster\" with FIPS enabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f04f00\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-z4f9jd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c1bb3da1-1029-43c0-9c00-a1c0774a9343\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c1bb3da1-1029-43c0-9c00-a1c0774a9343\",\n      \"name\": \"c1bb3da1-1029-43c0-9c00-a1c0774a9343\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:13.461257101Z\",\n      \"endTime\": \"2026-07-21T06:34:20.998111005Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"49ae1a1502691648b9691dac6fd4aed500111860d1d8540b4fb5f98f0f260d95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"90c1e458b7eafcb6f78d4c32d64f8d2e321ce0392834ddf6cf523aef2723da95","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015c7800\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a10f33f3-9e88-4363-bf17-d740cddc4e83\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a10f33f3-9e88-4363-bf17-d740cddc4e83\",\n      \"name\": \"a10f33f3-9e88-4363-bf17-d740cddc4e83\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:30:13.525087007Z\",\n      \"endTime\": \"2026-07-21T05:49:18.672814707Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015c7800\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a10f33f3-9e88-4363-bf17-d740cddc4e83\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a10f33f3-9e88-4363-bf17-d740cddc4e83\",\n      \"name\": \"a10f33f3-9e88-4363-bf17-d740cddc4e83\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:30:13.525087007Z\",\n      \"endTime\": \"2026-07-21T05:49:18.672814707Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4c0904df9f76985f3dc657693bfc022540741aa9735fd078d20cd9906123d63e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"2bcc421bad380fae66a24c3da5ba41037683155ae4e89d1bf10ed0a5ce05229f","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-m2949j with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00064ce60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-m2949j: failed waiting for cluster=\"np-version-upgrade-cluster-m2949j\" in resourcegroup=\"rg-np-version-upgrade-m2949j-gtt48s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ceee69c-4275-4d95-bd20-53f1b3209dfc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ceee69c-4275-4d95-bd20-53f1b3209dfc\",\n      \"name\": \"0ceee69c-4275-4d95-bd20-53f1b3209dfc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:48:05.373686621Z\",\n      \"endTime\": \"2026-07-21T08:07:14.188373989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-m2949j with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00064ce60\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-m2949j: failed waiting for cluster=\"np-version-upgrade-cluster-m2949j\" in resourcegroup=\"rg-np-version-upgrade-m2949j-gtt48s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ceee69c-4275-4d95-bd20-53f1b3209dfc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ceee69c-4275-4d95-bd20-53f1b3209dfc\",\n      \"name\": \"0ceee69c-4275-4d95-bd20-53f1b3209dfc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:48:05.373686621Z\",\n      \"endTime\": \"2026-07-21T08:07:14.188373989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5978f0f50be9afcfd40aab17f728c180964775131081e6ca68e897615ff67837","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"e5aae627e5423ff111b42a61998101a8f96b6b5c5b9d56e5f2a8ea110d53aa16","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-tg7z55\" with version 4.19.37 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8eb40\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-tg7z55: failed waiting for cluster=\"cluster-zstream-4-19-tg7z55\" in resourcegroup=\"rg-zstream-upgrade-4-19-wnk5t5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d14dd5f4-fbfa-4c37-b630-6607f591d25d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d14dd5f4-fbfa-4c37-b630-6607f591d25d\",\n      \"name\": \"d14dd5f4-fbfa-4c37-b630-6607f591d25d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:26:40.521192379Z\",\n      \"endTime\": \"2026-07-21T07:45:42.798808841Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-19-tg7z55\" with version 4.19.37 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8eb40\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-tg7z55: failed waiting for cluster=\"cluster-zstream-4-19-tg7z55\" in resourcegroup=\"rg-zstream-upgrade-4-19-wnk5t5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d14dd5f4-fbfa-4c37-b630-6607f591d25d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d14dd5f4-fbfa-4c37-b630-6607f591d25d\",\n      \"name\": \"d14dd5f4-fbfa-4c37-b630-6607f591d25d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:26:40.521192379Z\",\n      \"endTime\": \"2026-07-21T07:45:42.798808841Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"604965788578a2526f37e14db77b6041e0db24a135a3e6c68bb2522c18347432","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"ec53c2530d341327be6cba04ecfabef2505d44dcb0ec7138064837718114594c","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010e44a0\u003e: \n    failed to create HCP cluster ea-list7fm94z: failed waiting for cluster=\"ea-list7fm94z\" in resourcegroup=\"ea-list-4wl95k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e8ac5c37-9e27-4143-b32d-bdca417e4631\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e8ac5c37-9e27-4143-b32d-bdca417e4631\",\n      \"name\": \"e8ac5c37-9e27-4143-b32d-bdca417e4631\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:18:21.924991256Z\",\n      \"endTime\": \"2026-07-21T06:37:30.825960328Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:86]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010e44a0\u003e: \n    failed to create HCP cluster ea-list7fm94z: failed waiting for cluster=\"ea-list7fm94z\" in resourcegroup=\"ea-list-4wl95k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e8ac5c37-9e27-4143-b32d-bdca417e4631\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e8ac5c37-9e27-4143-b32d-bdca417e4631\",\n      \"name\": \"e8ac5c37-9e27-4143-b32d-bdca417e4631\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:18:21.924991256Z\",\n      \"endTime\": \"2026-07-21T06:37:30.825960328Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"65b46aebf5fffb76dda4f2d27823791cc2f18b1d24975bb5de9d9f70db08fa34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"9013e675989bd3312d95cfd426e5318859765876455d6850dc98ee4d503feff7","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"6804336cc2f0b5ca69c7978a32d5642a50721751145729e684e5b2f67a948864","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"9e265d39672bf2bb80b1948ac192f4778ccafb893e435adc5941967180736187","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create HCP cluster np-skip-minor-nwvcl6 at version 4.22.6\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000526520\u003e: \n    failed to create HCP cluster np-skip-minor-nwvcl6: failed waiting for cluster=\"np-skip-minor-nwvcl6\" in resourcegroup=\"rg-np-skip-minor-nwvcl6-fl8qmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f3024e89-9391-4c16-a923-94b70607bedd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f3024e89-9391-4c16-a923-94b70607bedd\",\n      \"name\": \"f3024e89-9391-4c16-a923-94b70607bedd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:35.484011576Z\",\n      \"endTime\": \"2026-07-21T07:21:51.982365687Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create HCP cluster np-skip-minor-nwvcl6 at version 4.22.6\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000526520\u003e: \n    failed to create HCP cluster np-skip-minor-nwvcl6: failed waiting for cluster=\"np-skip-minor-nwvcl6\" in resourcegroup=\"rg-np-skip-minor-nwvcl6-fl8qmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f3024e89-9391-4c16-a923-94b70607bedd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f3024e89-9391-4c16-a923-94b70607bedd\",\n      \"name\": \"f3024e89-9391-4c16-a923-94b70607bedd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:35.484011576Z\",\n      \"endTime\": \"2026-07-21T07:21:51.982365687Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"683717850b15578d442801579720f4d7c8976d3b3161388bf5b8bbc3c993367c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"f38b7a43d956e10a30e734485ca2f145486fc2357c54ed13974c113dfd5683b2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a50140\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mkrqwr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/059fab94-4fa7-4c9e-96a8-a340618ff9f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/059fab94-4fa7-4c9e-96a8-a340618ff9f0\",\n      \"name\": \"059fab94-4fa7-4c9e-96a8-a340618ff9f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:30:56.636621797Z\",\n      \"endTime\": \"2026-07-21T05:49:58.672166784Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a50140\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mkrqwr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/059fab94-4fa7-4c9e-96a8-a340618ff9f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/059fab94-4fa7-4c9e-96a8-a340618ff9f0\",\n      \"name\": \"059fab94-4fa7-4c9e-96a8-a340618ff9f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:30:56.636621797Z\",\n      \"endTime\": \"2026-07-21T05:49:58.672166784Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6f379756752de5720a81d52d44dbc5b0c3829ce0fa508a7384ec280e7bdd70a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"007cf65197ae2bd9e8f2f7404c2533a5b9e03fc988753ffd47ce5e0c72d520e1","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7c300\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-n5wk42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"name\": \"b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:32:09.443545349Z\",\n      \"endTime\": \"2026-07-21T05:51:18.672656967Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7c300\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-n5wk42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"name\": \"b7f26b4b-723b-4092-913d-37ce64f501e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:32:09.443545349Z\",\n      \"endTime\": \"2026-07-21T05:51:18.672656967Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6fae2514c3305b4d3a9ba22a8838a6dfdee3e2b2606e4efa9c203cc153666ee5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"4443d87be4b037676b273fcbae9fe676e5dc3fc90bee3f49b560fe421976d2ea","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-4k2tqj\" with version 4.20.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f00e40\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-4k2tqj: failed waiting for cluster=\"cluster-zstream-4-20-4k2tqj\" in resourcegroup=\"rg-zstream-upgrade-4-20-m682jg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/138b4d17-82d0-4e79-a487-9a38a5bba931\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/138b4d17-82d0-4e79-a487-9a38a5bba931\",\n      \"name\": \"138b4d17-82d0-4e79-a487-9a38a5bba931\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:47:54.939497024Z\",\n      \"endTime\": \"2026-07-21T08:06:54.187475466Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-20-4k2tqj\" with version 4.20.29 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f00e40\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-4k2tqj: failed waiting for cluster=\"cluster-zstream-4-20-4k2tqj\" in resourcegroup=\"rg-zstream-upgrade-4-20-m682jg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/138b4d17-82d0-4e79-a487-9a38a5bba931\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/138b4d17-82d0-4e79-a487-9a38a5bba931\",\n      \"name\": \"138b4d17-82d0-4e79-a487-9a38a5bba931\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:47:54.939497024Z\",\n      \"endTime\": \"2026-07-21T08:06:54.187475466Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"716cbde27b20a5857b0044bea0c3a5b4e04f4a835e1c9e7ac0dedf2d65808c52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"f13cea685ea27d03325fa3981a4a87503b664077cee826714d71eb7f40b664eb","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:312]: failed to create HCP cluster np-noedge-pnzbd8\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00059c1a0\u003e: \n    failed to create HCP cluster np-noedge-pnzbd8: failed waiting for cluster=\"np-noedge-pnzbd8\" in resourcegroup=\"rg-np-noedge-pnzbd8-qqtl7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7f7ed66a-02c7-4db9-a33e-9e09233bc1ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7f7ed66a-02c7-4db9-a33e-9e09233bc1ed\",\n      \"name\": \"7f7ed66a-02c7-4db9-a33e-9e09233bc1ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:23.931959531Z\",\n      \"endTime\": \"2026-07-21T07:21:31.785223415Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:312]: failed to create HCP cluster np-noedge-pnzbd8\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00059c1a0\u003e: \n    failed to create HCP cluster np-noedge-pnzbd8: failed waiting for cluster=\"np-noedge-pnzbd8\" in resourcegroup=\"rg-np-noedge-pnzbd8-qqtl7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7f7ed66a-02c7-4db9-a33e-9e09233bc1ed\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/7f7ed66a-02c7-4db9-a33e-9e09233bc1ed\",\n      \"name\": \"7f7ed66a-02c7-4db9-a33e-9e09233bc1ed\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:23.931959531Z\",\n      \"endTime\": \"2026-07-21T07:21:31.785223415Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"88f770f79ac6e01748d1f45261ef5f77c4cd705d2a62e6b0f1744bdf6ce3d383","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"5e36b091c1c34d3b51e47b9b6dd43f58cd9824f8e9079daacb163895f2200c4b","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96260\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hlqwlg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b808ec30-96b2-4512-8fcf-d71f58e3240c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b808ec30-96b2-4512-8fcf-d71f58e3240c\",\n      \"name\": \"b808ec30-96b2-4512-8fcf-d71f58e3240c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:30:59.728003362Z\",\n      \"endTime\": \"2026-07-21T05:50:08.662432514Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e96260\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hlqwlg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b808ec30-96b2-4512-8fcf-d71f58e3240c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/b808ec30-96b2-4512-8fcf-d71f58e3240c\",\n      \"name\": \"b808ec30-96b2-4512-8fcf-d71f58e3240c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:30:59.728003362Z\",\n      \"endTime\": \"2026-07-21T05:50:08.662432514Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9547b0d4a636fa698aefb13a96ae41277468d3b8aceeb0809f2fac2f549d846b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"3e96a226b21e30dc74c54fd3b273577eb22b5f774239fdc15a8148abede3f046","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e92160\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-qrl6bq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c7c0892b-200d-4ce9-9ab5-f4ae47b73b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c7c0892b-200d-4ce9-9ab5-f4ae47b73b53\",\n      \"name\": \"c7c0892b-200d-4ce9-9ab5-f4ae47b73b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:28:52.270388155Z\",\n      \"endTime\": \"2026-07-21T05:47:58.66828078Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e92160\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-qrl6bq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c7c0892b-200d-4ce9-9ab5-f4ae47b73b53\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c7c0892b-200d-4ce9-9ab5-f4ae47b73b53\",\n      \"name\": \"c7c0892b-200d-4ce9-9ab5-f4ae47b73b53\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:28:52.270388155Z\",\n      \"endTime\": \"2026-07-21T05:47:58.66828078Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"98c555e4531a9e5a7628c22e4c2daa75f6861b195cb7803ac471ff37342f9474","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"821c3738d4a7273a8d57242e8b33fc7bfe75d51cb30bd8fecd66aa86c3d0955f","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c8ea00\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-zrnnwr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8d90e467-739c-4b3a-98f0-b5dfb2760acc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8d90e467-739c-4b3a-98f0-b5dfb2760acc\",\n      \"name\": \"8d90e467-739c-4b3a-98f0-b5dfb2760acc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:16:55.071738854Z\",\n      \"endTime\": \"2026-07-21T06:36:01.032275395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c8ea00\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-zrnnwr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8d90e467-739c-4b3a-98f0-b5dfb2760acc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8d90e467-739c-4b3a-98f0-b5dfb2760acc\",\n      \"name\": \"8d90e467-739c-4b3a-98f0-b5dfb2760acc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:16:55.071738854Z\",\n      \"endTime\": \"2026-07-21T06:36:01.032275395Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9cce55d68a684793326be3df1534f61fdf0771604623e6cf0fda701c7c863dd5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"852f63db6e99b0fc70dc00b3651ff3a8d766a89f489846e52cb8b55f8af962d5","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00017c220\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-nrtk2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ae62610-3173-452f-a315-1330d45b2f62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ae62610-3173-452f-a315-1330d45b2f62\",\n      \"name\": \"0ae62610-3173-452f-a315-1330d45b2f62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:11.807853042Z\",\n      \"endTime\": \"2026-07-21T06:34:11.02999724Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00017c220\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-nrtk2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ae62610-3173-452f-a315-1330d45b2f62\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0ae62610-3173-452f-a315-1330d45b2f62\",\n      \"name\": \"0ae62610-3173-452f-a315-1330d45b2f62\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:11.807853042Z\",\n      \"endTime\": \"2026-07-21T06:34:11.02999724Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9eb7f1b921b5a5c1d79e3373415b48df280cdb333a7566f15c5b55da2bf43488","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"7f31b126a97f6a9ae3ed0e80c1fe187e4aa077e38f647f414370c69a5be23c1f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-fxbfnk/cluster-candidate-4-22-sz92br should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010ca060\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-sz92br: failed waiting for cluster=\"cluster-candidate-4-22-sz92br\" in resourcegroup=\"rg-candidate-4-22-fxbfnk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90988457-2dc2-4bce-802b-6ce46b7a86c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90988457-2dc2-4bce-802b-6ce46b7a86c5\",\n      \"name\": \"90988457-2dc2-4bce-802b-6ce46b7a86c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:12.933420696Z\",\n      \"endTime\": \"2026-07-21T06:34:20.829885185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-fxbfnk/cluster-candidate-4-22-sz92br should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010ca060\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-sz92br: failed waiting for cluster=\"cluster-candidate-4-22-sz92br\" in resourcegroup=\"rg-candidate-4-22-fxbfnk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90988457-2dc2-4bce-802b-6ce46b7a86c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/90988457-2dc2-4bce-802b-6ce46b7a86c5\",\n      \"name\": \"90988457-2dc2-4bce-802b-6ce46b7a86c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:12.933420696Z\",\n      \"endTime\": \"2026-07-21T06:34:20.829885185Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9f84852da5a1130cb64d597ce42a45d3c96fc310efa196a7584a6e4eb00bf3a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"0da9a6549c113196b6efc7951a0a18523289a97e22ff686f54dfd7e467f095b4","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009721e0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hkl2lz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6eb0c22f-0dc5-44a9-83a8-eaba106b4396\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6eb0c22f-0dc5-44a9-83a8-eaba106b4396\",\n      \"name\": \"6eb0c22f-0dc5-44a9-83a8-eaba106b4396\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:48:05.336144366Z\",\n      \"endTime\": \"2026-07-21T08:07:14.180527663Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009721e0\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hkl2lz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6eb0c22f-0dc5-44a9-83a8-eaba106b4396\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/6eb0c22f-0dc5-44a9-83a8-eaba106b4396\",\n      \"name\": \"6eb0c22f-0dc5-44a9-83a8-eaba106b4396\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:48:05.336144366Z\",\n      \"endTime\": \"2026-07-21T08:07:14.180527663Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a1ec9f2786f0d58b39fc1ecb7cbb58502b9b924be93fa536b5d7a1db902c512e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"ef147d5fb1533c538e021dfaed880dd33e34dc663172abf3e4ea564a4b05976c","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009fea40\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-lz8xd6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/2f540547-dd16-41e8-b909-0c7a6c3f4ae4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/2f540547-dd16-41e8-b909-0c7a6c3f4ae4\",\n      \"name\": \"2f540547-dd16-41e8-b909-0c7a6c3f4ae4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:40:34.566725422Z\",\n      \"endTime\": \"2026-07-21T06:59:51.656594475Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009fea40\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-lz8xd6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/2f540547-dd16-41e8-b909-0c7a6c3f4ae4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/2f540547-dd16-41e8-b909-0c7a6c3f4ae4\",\n      \"name\": \"2f540547-dd16-41e8-b909-0c7a6c3f4ae4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:40:34.566725422Z\",\n      \"endTime\": \"2026-07-21T06:59:51.656594475Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a5c911a16ef41446daa2943901ac4c76ad606d3dd2f70998783ed6c9e90967d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"bc9487be47906f8adc0829bfaeb7891f9516a919d5c701126df8d3c1a2732e0a","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000984da0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-dj76gg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4752aea8-b3f2-4348-a67b-2bb1733e7c9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4752aea8-b3f2-4348-a67b-2bb1733e7c9c\",\n      \"name\": \"4752aea8-b3f2-4348-a67b-2bb1733e7c9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:04:16.702071325Z\",\n      \"endTime\": \"2026-07-21T07:23:21.792207857Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000984da0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-dj76gg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4752aea8-b3f2-4348-a67b-2bb1733e7c9c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4752aea8-b3f2-4348-a67b-2bb1733e7c9c\",\n      \"name\": \"4752aea8-b3f2-4348-a67b-2bb1733e7c9c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:04:16.702071325Z\",\n      \"endTime\": \"2026-07-21T07:23:21.792207857Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"acdcb854710746b3c18e822a4f33cd262a2476af7ae9cc09aaca8077728a943d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"db6c0aad1a7eb3447967f2ef43e12a6e5c1c8c6982e603975af0a727456df14d","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00098e570\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-t84nc4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-t84nc4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00098e570\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-t84nc4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-t84nc4, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-t84nc4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-t84nc4, error: context deadline exceeded"}},{"row_id":"ad3bd06ed5061ab5827916c890984e6725112511ed635db2be17451a663ca0ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"43e89e3345bb7d556171b9d391dc2538b1e10d9256a72fed3572449c69e0913f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-21-j5cftg\" with version 4.21.24 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d92be0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-j5cftg: failed waiting for cluster=\"cluster-zstream-4-21-j5cftg\" in resourcegroup=\"rg-zstream-upgrade-4-21-lcdrlx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/79c4e439-8a7f-44e1-b16d-11b9a2a2440a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/79c4e439-8a7f-44e1-b16d-11b9a2a2440a\",\n      \"name\": \"79c4e439-8a7f-44e1-b16d-11b9a2a2440a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:50.917965891Z\",\n      \"endTime\": \"2026-07-21T06:35:00.842728957Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: failed to create HCP cluster \"cluster-zstream-4-21-j5cftg\" with version 4.21.24 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d92be0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-j5cftg: failed waiting for cluster=\"cluster-zstream-4-21-j5cftg\" in resourcegroup=\"rg-zstream-upgrade-4-21-lcdrlx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/79c4e439-8a7f-44e1-b16d-11b9a2a2440a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/79c4e439-8a7f-44e1-b16d-11b9a2a2440a\",\n      \"name\": \"79c4e439-8a7f-44e1-b16d-11b9a2a2440a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:50.917965891Z\",\n      \"endTime\": \"2026-07-21T06:35:00.842728957Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b3c282ba9b17482c42c152bb6d997d0996bf33730d8d7f8784da7be5caff57a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"6ed96bf3c4348483e2e6c1462330c35308c01990058d8eabde900f6ec36cdfed","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:150]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005a70c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-h4rmfh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5183ea76-06ad-42a3-8819-9a4756655798\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5183ea76-06ad-42a3-8819-9a4756655798\",\n      \"name\": \"5183ea76-06ad-42a3-8819-9a4756655798\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:30:01.155681755Z\",\n      \"endTime\": \"2026-07-21T05:49:08.65841302Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:150]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005a70c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-h4rmfh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5183ea76-06ad-42a3-8819-9a4756655798\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5183ea76-06ad-42a3-8819-9a4756655798\",\n      \"name\": \"5183ea76-06ad-42a3-8819-9a4756655798\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:30:01.155681755Z\",\n      \"endTime\": \"2026-07-21T05:49:08.65841302Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b4bf5f486c279740c33aaf3dde29936e5f889433a064be02bcca755e6c58a1e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"6d8a9aa9e4af877217edf30f3c6c25e1a8dd51ae566327ed7d560207914b8d6b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-20-76bnn7/cluster-candidate-4-20-sf7b82 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c78220\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-sf7b82: failed waiting for cluster=\"cluster-candidate-4-20-sf7b82\" in resourcegroup=\"rg-candidate-4-20-76bnn7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e39d5ee5-7775-488d-a851-d0956ee53952\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e39d5ee5-7775-488d-a851-d0956ee53952\",\n      \"name\": \"e39d5ee5-7775-488d-a851-d0956ee53952\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:35.852655553Z\",\n      \"endTime\": \"2026-07-21T07:21:41.786817745Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-20-76bnn7/cluster-candidate-4-20-sf7b82 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c78220\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-sf7b82: failed waiting for cluster=\"cluster-candidate-4-20-sf7b82\" in resourcegroup=\"rg-candidate-4-20-76bnn7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e39d5ee5-7775-488d-a851-d0956ee53952\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e39d5ee5-7775-488d-a851-d0956ee53952\",\n      \"name\": \"e39d5ee5-7775-488d-a851-d0956ee53952\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:35.852655553Z\",\n      \"endTime\": \"2026-07-21T07:21:41.786817745Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b688a7a6e0b46dab0e6f609ba26b177240f7fbefafbccfa54df0ca419dedb5a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"441d8fa32423755b824b03f0ed9756d2f6f572771ded2eeacd532359e6529d9c","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009cc3a0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-bl679h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/60e90de0-64d3-49ba-b122-4638e3cfc8b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/60e90de0-64d3-49ba-b122-4638e3cfc8b8\",\n      \"name\": \"60e90de0-64d3-49ba-b122-4638e3cfc8b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:32.941241255Z\",\n      \"endTime\": \"2026-07-21T07:20:41.78673603Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:111]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009cc3a0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-bl679h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/60e90de0-64d3-49ba-b122-4638e3cfc8b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/60e90de0-64d3-49ba-b122-4638e3cfc8b8\",\n      \"name\": \"60e90de0-64d3-49ba-b122-4638e3cfc8b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:32.941241255Z\",\n      \"endTime\": \"2026-07-21T07:20:41.78673603Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bcaf40e9d6e1f9fa23ab9eafd679d18654f072a8ab32c80c665e5d205b1dd590","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"832df04ff0266c019a92ac741e0994e1c347d1dcd016e6114a9d7a19a502c011","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c04660\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-sfh69z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/75a023e5-a852-418a-afc7-39f8e48cc219\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/75a023e5-a852-418a-afc7-39f8e48cc219\",\n      \"name\": \"75a023e5-a852-418a-afc7-39f8e48cc219\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:32.928242235Z\",\n      \"endTime\": \"2026-07-21T07:20:32.122372644Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c04660\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-sfh69z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/75a023e5-a852-418a-afc7-39f8e48cc219\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/75a023e5-a852-418a-afc7-39f8e48cc219\",\n      \"name\": \"75a023e5-a852-418a-afc7-39f8e48cc219\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:32.928242235Z\",\n      \"endTime\": \"2026-07-21T07:20:32.122372644Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bea4be93b59a73b82bdac6de3938f3dd79c2c3718fba189aa18f2ea6ef95239e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"6526809d46dcfa990c8305b84fe9819b03f13654c0fe7e2ce4e680af90a63107","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b461a0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-87l7qc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c7909bdc-2ccd-4ddd-90c9-23e8ae8f68c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c7909bdc-2ccd-4ddd-90c9-23e8ae8f68c8\",\n      \"name\": \"c7909bdc-2ccd-4ddd-90c9-23e8ae8f68c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:28:53.740749563Z\",\n      \"endTime\": \"2026-07-21T05:48:08.714306607Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b461a0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-87l7qc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c7909bdc-2ccd-4ddd-90c9-23e8ae8f68c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c7909bdc-2ccd-4ddd-90c9-23e8ae8f68c8\",\n      \"name\": \"c7909bdc-2ccd-4ddd-90c9-23e8ae8f68c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:28:53.740749563Z\",\n      \"endTime\": \"2026-07-21T05:48:08.714306607Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c217be59f9e40ecf404646546eb133c3eedb060bcd45b5071fd025f3a685de9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"3ea767832316ee9b0fc7ee7b62eac613c76d48aa08765715d31b65da9ceba713","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f02e60\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-dsmmrz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/385a5798-c4af-42f4-a3ac-ce229c89fdec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/385a5798-c4af-42f4-a3ac-ce229c89fdec\",\n      \"name\": \"385a5798-c4af-42f4-a3ac-ce229c89fdec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:28:53.016439742Z\",\n      \"endTime\": \"2026-07-21T05:47:58.664343617Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:89]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f02e60\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-dsmmrz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/385a5798-c4af-42f4-a3ac-ce229c89fdec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/385a5798-c4af-42f4-a3ac-ce229c89fdec\",\n      \"name\": \"385a5798-c4af-42f4-a3ac-ce229c89fdec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:28:53.016439742Z\",\n      \"endTime\": \"2026-07-21T05:47:58.664343617Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c218fb88338bfe54719d7cba3a64b3c5fd12517d8fbc74e6ec31d523de2eb863","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"6ce216fca325d3fb38de5d65149fc99e7777223055b036d53766b1a5ebc53e24","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000870b20\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-d2gjtj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3499e1f-2cc7-40b2-8b46-60fa685aa449\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3499e1f-2cc7-40b2-8b46-60fa685aa449\",\n      \"name\": \"e3499e1f-2cc7-40b2-8b46-60fa685aa449\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:28:52.447378332Z\",\n      \"endTime\": \"2026-07-21T05:47:58.661162265Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000870b20\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-d2gjtj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3499e1f-2cc7-40b2-8b46-60fa685aa449\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e3499e1f-2cc7-40b2-8b46-60fa685aa449\",\n      \"name\": \"e3499e1f-2cc7-40b2-8b46-60fa685aa449\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:28:52.447378332Z\",\n      \"endTime\": \"2026-07-21T05:47:58.661162265Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c99a3571ffaff856f7cc8daf2ab76779c0afa5b8a89a4a51431285d6300a1abe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"d87b62a2d298d16c9744cb8fdfcc55133bcde0f9346809cb0c3f963f838fdce4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-fjf9wl/cluster-candidate-4-21-cpvphl should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aa24c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-cpvphl: failed waiting for cluster=\"cluster-candidate-4-21-cpvphl\" in resourcegroup=\"rg-candidate-4-21-fjf9wl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fb608369-19a6-4d8d-8db4-d2a1822f407e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fb608369-19a6-4d8d-8db4-d2a1822f407e\",\n      \"name\": \"fb608369-19a6-4d8d-8db4-d2a1822f407e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:24.383919652Z\",\n      \"endTime\": \"2026-07-21T06:34:31.001952791Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-fjf9wl/cluster-candidate-4-21-cpvphl should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aa24c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-cpvphl: failed waiting for cluster=\"cluster-candidate-4-21-cpvphl\" in resourcegroup=\"rg-candidate-4-21-fjf9wl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fb608369-19a6-4d8d-8db4-d2a1822f407e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fb608369-19a6-4d8d-8db4-d2a1822f407e\",\n      \"name\": \"fb608369-19a6-4d8d-8db4-d2a1822f407e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:15:24.383919652Z\",\n      \"endTime\": \"2026-07-21T06:34:31.001952791Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cc10a844d68da1ac915c974cda3052fe770852baadad20d701e39532775d841d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"a207a9f75a81edc168e301bf11e3bc2b58753e11ec0d8aa2c76c09742fb52bb5","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-zlzcng with version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0e5c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-zlzcng: failed waiting for cluster=\"np-version-upgrade-cluster-zlzcng\" in resourcegroup=\"rg-np-version-upgrade-zlzcng-7m6h5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b607dc2-506a-4438-ab17-d5486303ec89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b607dc2-506a-4438-ab17-d5486303ec89\",\n      \"name\": \"1b607dc2-506a-4438-ab17-d5486303ec89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:33.143154715Z\",\n      \"endTime\": \"2026-07-21T07:21:51.776659374Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-zlzcng with version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0e5c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-zlzcng: failed waiting for cluster=\"np-version-upgrade-cluster-zlzcng\" in resourcegroup=\"rg-np-version-upgrade-zlzcng-7m6h5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b607dc2-506a-4438-ab17-d5486303ec89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b607dc2-506a-4438-ab17-d5486303ec89\",\n      \"name\": \"1b607dc2-506a-4438-ab17-d5486303ec89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:02:33.143154715Z\",\n      \"endTime\": \"2026-07-21T07:21:51.776659374Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d16a428b0fe0ed94ce590d5f13634ce3219b324c67121e4cf0b10b7405f73cb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"daeb2fe029265bf5afe8f986c4a0715c02ec32c755e57221ef29cccb0baf5b77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"c51c4d2e9aa454582bfa293dcddf09e5e62340b8d5506cbcd60056b871eff4fe","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b0140\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-md4n7c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e64d1c93-09bc-4b5f-8170-b58cd7e0d983\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e64d1c93-09bc-4b5f-8170-b58cd7e0d983\",\n      \"name\": \"e64d1c93-09bc-4b5f-8170-b58cd7e0d983\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:29:05.966593099Z\",\n      \"endTime\": \"2026-07-21T05:48:08.714138504Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b0140\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-md4n7c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e64d1c93-09bc-4b5f-8170-b58cd7e0d983\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e64d1c93-09bc-4b5f-8170-b58cd7e0d983\",\n      \"name\": \"e64d1c93-09bc-4b5f-8170-b58cd7e0d983\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T05:29:05.966593099Z\",\n      \"endTime\": \"2026-07-21T05:48:08.714138504Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"de21cee873d6037c93493876024efbff20c3704565d0ae4b9919f8addff8be89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"cbd57e43d8472457315665ef7b2a188b7fcdec695f9e63105ea81d3a101c61c2","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d00820\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-9f8gj7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/562d3403-3ef2-4a28-bb04-d52d2bcd06e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/562d3403-3ef2-4a28-bb04-d52d2bcd06e3\",\n      \"name\": \"562d3403-3ef2-4a28-bb04-d52d2bcd06e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:22.952319357Z\",\n      \"endTime\": \"2026-07-21T07:20:22.172240034Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d00820\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-9f8gj7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/562d3403-3ef2-4a28-bb04-d52d2bcd06e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/562d3403-3ef2-4a28-bb04-d52d2bcd06e3\",\n      \"name\": \"562d3403-3ef2-4a28-bb04-d52d2bcd06e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:22.952319357Z\",\n      \"endTime\": \"2026-07-21T07:20:22.172240034Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ea1e49962820ccb9c20855f7a7c06547ecf6510280e85e5654114eb2ac090692","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"e48b650eeda4387d00133fbd7828d69f6b4d3d2563d1d298b5bd6c5d83854933","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c8e680\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-9x494z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e8206088-31df-4551-abb8-4d4a5b27f702\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e8206088-31df-4551-abb8-4d4a5b27f702\",\n      \"name\": \"e8206088-31df-4551-abb8-4d4a5b27f702\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:33.879899304Z\",\n      \"endTime\": \"2026-07-21T07:20:41.796833696Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c8e680\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-9x494z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e8206088-31df-4551-abb8-4d4a5b27f702\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e8206088-31df-4551-abb8-4d4a5b27f702\",\n      \"name\": \"e8206088-31df-4551-abb8-4d4a5b27f702\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:33.879899304Z\",\n      \"endTime\": \"2026-07-21T07:20:41.796833696Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ee6de66d22d2309bf8f58cc2398ff605e4a1a98273f6fc6e74027f15805d2acc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"a45a7c891b05805a6f24cd2392de01dd72e32ba8f0885fbf071bdb3f29969caf","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-s4hhzw with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7e840\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-s4hhzw: failed waiting for cluster=\"np-version-upgrade-cluster-s4hhzw\" in resourcegroup=\"rg-np-version-upgrade-s4hhzw-25q4g2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a75198c9-1b57-4734-8578-7e5eb8acd69f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a75198c9-1b57-4734-8578-7e5eb8acd69f\",\n      \"name\": \"a75198c9-1b57-4734-8578-7e5eb8acd69f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:56.859771171Z\",\n      \"endTime\": \"2026-07-21T07:21:01.79981848Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: failed to create HCP cluster np-version-upgrade-cluster-s4hhzw with version 4.21.25\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a7e840\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-s4hhzw: failed waiting for cluster=\"np-version-upgrade-cluster-s4hhzw\" in resourcegroup=\"rg-np-version-upgrade-s4hhzw-25q4g2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a75198c9-1b57-4734-8578-7e5eb8acd69f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/a75198c9-1b57-4734-8578-7e5eb8acd69f\",\n      \"name\": \"a75198c9-1b57-4734-8578-7e5eb8acd69f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:56.859771171Z\",\n      \"endTime\": \"2026-07-21T07:21:01.79981848Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eff6be2bbd1342cbba5f0fc91793d12abda89c95fc7b786c8fa9af3ac43fa072","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"ea5c3bad22c7b0d89ec624594917a1c4cecef86c831081ab9fa716248d9a80d8","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ddc220\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-g8czq2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d3b31812-d979-4f34-8fec-c61eea416083\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d3b31812-d979-4f34-8fec-c61eea416083\",\n      \"name\": \"d3b31812-d979-4f34-8fec-c61eea416083\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:32.398689049Z\",\n      \"endTime\": \"2026-07-21T07:20:32.118632582Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ddc220\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-g8czq2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d3b31812-d979-4f34-8fec-c61eea416083\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d3b31812-d979-4f34-8fec-c61eea416083\",\n      \"name\": \"d3b31812-d979-4f34-8fec-c61eea416083\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T07:01:32.398689049Z\",\n      \"endTime\": \"2026-07-21T07:20:32.118632582Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f0cf2e9f25374ffcc9cfac4afb6e49a6eaf112b6e9b4fb4f28681dce6f72af8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079434512579694592","occurred_at":"2026-07-21T05:13:16Z","failed_at":"e2e","signature_id":"e415c01a07919d6141e32036bedd7d2780b319450f7bd4c310fde9e4dd4fdfea","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:181]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-w44m4b\" with install version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b948a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-w44m4b: failed waiting for cluster=\"cp-ystream-upgrade-4-21-w44m4b\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-xfn5j7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/148f6fb2-1791-4c33-a5de-b9a2feaafcbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/148f6fb2-1791-4c33-a5de-b9a2feaafcbe\",\n      \"name\": \"148f6fb2-1791-4c33-a5de-b9a2feaafcbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:16:41.482064322Z\",\n      \"endTime\": \"2026-07-21T06:35:50.815351312Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:181]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-w44m4b\" with install version 4.20.30\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b948a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-w44m4b: failed waiting for cluster=\"cp-ystream-upgrade-4-21-w44m4b\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-xfn5j7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/148f6fb2-1791-4c33-a5de-b9a2feaafcbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/148f6fb2-1791-4c33-a5de-b9a2feaafcbe\",\n      \"name\": \"148f6fb2-1791-4c33-a5de-b9a2feaafcbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-21T06:16:41.482064322Z\",\n      \"endTime\": \"2026-07-21T06:35:50.815351312Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":45,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","460f3a8c506a01a0b3bec28418448a26cbb3f6be39d0a43e0d724adf1f0ff118","f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b"],"matched_occurrences":45,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2079386059505930240","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-21T02:00:44Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2079369848160260096","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-21T00:56:19Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}