{"meta":{"date":"2026-08-02","timezone":"UTC","generated_at":"2026-08-26T10:54:04Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":3,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084051647016734720","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-02T23:00:07Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0757425933998480d4eeba254365eb25f27261568e6e1cb2ab79bbe473136c20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2084051647016734720","occurred_at":"2026-08-02T23:00:07Z","failed_at":"e2e","signature_id":"9f0e17695dbb84bcb7660491c012000cf531902427748b6227ba96eff615d5d8","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:137]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24a00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-9ngxf9zlhblm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3463e632-f100-45af-9f71-fc69cdf2dd4c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3463e632-f100-45af-9f71-fc69cdf2dd4c\",\n      \"name\": \"3463e632-f100-45af-9f71-fc69cdf2dd4c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T23:16:20.627905758Z\",\n      \"endTime\": \"2026-08-02T23:35:28.37866899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:137]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24a00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-9ngxf9zlhblm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3463e632-f100-45af-9f71-fc69cdf2dd4c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3463e632-f100-45af-9f71-fc69cdf2dd4c\",\n      \"name\": \"3463e632-f100-45af-9f71-fc69cdf2dd4c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T23:16:20.627905758Z\",\n      \"endTime\": \"2026-08-02T23:35:28.37866899Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-02T09:15:40Z"},"failed_at":["e2e"],"failed_test_count":38,"occurrences":[{"row_id":"01e9fa1ccd59f16a2ae938d62bf154efa03e71394d478154fc990cbfefce45cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"fff0e5b06a86af9a09ba19f4ff37b1e7c499c4333027af36c522fa247ad2f3c2","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d90900\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-7tzmlfvcvjz2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8e7f6e9d-54e9-43d6-9057-7304d01b9584\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8e7f6e9d-54e9-43d6-9057-7304d01b9584\",\n      \"name\": \"8e7f6e9d-54e9-43d6-9057-7304d01b9584\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:18.700863215Z\",\n      \"endTime\": \"2026-08-02T09:53:19.940689234Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d90900\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-7tzmlfvcvjz2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8e7f6e9d-54e9-43d6-9057-7304d01b9584\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8e7f6e9d-54e9-43d6-9057-7304d01b9584\",\n      \"name\": \"8e7f6e9d-54e9-43d6-9057-7304d01b9584\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:18.700863215Z\",\n      \"endTime\": \"2026-08-02T09:53:19.940689234Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"027b133ff4eed72f26ac602b440b5670c1447362a468e6dc7f9e8a6b4381b457","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"4dac8a5c315ca81423f117e3fca48c1fa722a6c62acb39a85e9651b3ff831f22","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009404a0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4k4dbkr4s254\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4c3c5ea4-015b-4faa-988b-ca92a5589f78\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4c3c5ea4-015b-4faa-988b-ca92a5589f78\",\n      \"name\": \"4c3c5ea4-015b-4faa-988b-ca92a5589f78\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:01.642064318Z\",\n      \"endTime\": \"2026-08-02T09:53:20.129747727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009404a0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-4k4dbkr4s254\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4c3c5ea4-015b-4faa-988b-ca92a5589f78\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4c3c5ea4-015b-4faa-988b-ca92a5589f78\",\n      \"name\": \"4c3c5ea4-015b-4faa-988b-ca92a5589f78\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:01.642064318Z\",\n      \"endTime\": \"2026-08-02T09:53:20.129747727Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"02ebc012710cff6abff1e75cf8965dae2f6ec574edd5b87d509641d8fb31d854","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"73397df6b0d9f6da82d4449e3fa8ea2eccb205f8dd99d6726b907bd5b53a5a64","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-hj85vs at version 4.22.8\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd93c0\u003e: \n    failed to create HCP cluster np-skip-minor-hj85vs: failed waiting for cluster=\"np-skip-minor-hj85vs\" in resourcegroup=\"rg-np-skip-minor-hj85vs-5sslvjsdhbbb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/79102d9d-d8f7-4787-8796-65c7af655254\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/79102d9d-d8f7-4787-8796-65c7af655254\",\n      \"name\": \"79102d9d-d8f7-4787-8796-65c7af655254\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:46:52.48500524Z\",\n      \"endTime\": \"2026-08-02T11:05:53.701127939Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-hj85vs at version 4.22.8\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd93c0\u003e: \n    failed to create HCP cluster np-skip-minor-hj85vs: failed waiting for cluster=\"np-skip-minor-hj85vs\" in resourcegroup=\"rg-np-skip-minor-hj85vs-5sslvjsdhbbb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/79102d9d-d8f7-4787-8796-65c7af655254\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/79102d9d-d8f7-4787-8796-65c7af655254\",\n      \"name\": \"79102d9d-d8f7-4787-8796-65c7af655254\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:46:52.48500524Z\",\n      \"endTime\": \"2026-08-02T11:05:53.701127939Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0a5cf74468625405c050f7fbdbeeadfb90055d5d3af6dc9ab95a4de2b92e13b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"5fb90d3ae62f48b86e0520e6fbb53bb0d6a7df71901e6e1dd5d13262315140b4","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00099e6a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-h88dk8ndppfm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6dbae615-f8d0-4154-ac6c-998d81f7b84d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6dbae615-f8d0-4154-ac6c-998d81f7b84d\",\n      \"name\": \"6dbae615-f8d0-4154-ac6c-998d81f7b84d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:33:48.731562196Z\",\n      \"endTime\": \"2026-08-02T09:52:59.947253927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00099e6a0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-h88dk8ndppfm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6dbae615-f8d0-4154-ac6c-998d81f7b84d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6dbae615-f8d0-4154-ac6c-998d81f7b84d\",\n      \"name\": \"6dbae615-f8d0-4154-ac6c-998d81f7b84d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:33:48.731562196Z\",\n      \"endTime\": \"2026-08-02T09:52:59.947253927Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0a97bc999d47f9283d752dab3f07331d926fd00de9eb6c111d9866beb72b4306","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"f9d091a59a7901e6df23ca61e4790b655188be668b236e3780226d657a1365e1","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-b59ll2\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ddd1e0\u003e: \n    failed to create HCP cluster np-noedge-b59ll2: failed waiting for cluster=\"np-noedge-b59ll2\" in resourcegroup=\"rg-np-noedge-b59ll2-p7jkwbdc422s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1654762e-f7e5-4618-a61f-b73443f5f202\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1654762e-f7e5-4618-a61f-b73443f5f202\",\n      \"name\": \"1654762e-f7e5-4618-a61f-b73443f5f202\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:26:18.049472465Z\",\n      \"endTime\": \"2026-08-02T10:45:22.746370197Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-b59ll2\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ddd1e0\u003e: \n    failed to create HCP cluster np-noedge-b59ll2: failed waiting for cluster=\"np-noedge-b59ll2\" in resourcegroup=\"rg-np-noedge-b59ll2-p7jkwbdc422s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1654762e-f7e5-4618-a61f-b73443f5f202\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1654762e-f7e5-4618-a61f-b73443f5f202\",\n      \"name\": \"1654762e-f7e5-4618-a61f-b73443f5f202\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:26:18.049472465Z\",\n      \"endTime\": \"2026-08-02T10:45:22.746370197Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0ce4aee9408a3d224f8cdd0efea29264193b220fb6b43a3730980c3c513c305d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"0e52bbc015cebdfae2cb2ec73bce70b50ff7676290e8d16faea3ee36e628737a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"9b8250b9774580074602a315c723a89e946d1b5bf10a47c683301668aec3a81d","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:151]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019052c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wmrgqh8gzn96\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0fbe47c0-96cd-4379-9f15-8327e095b80e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0fbe47c0-96cd-4379-9f15-8327e095b80e\",\n      \"name\": \"0fbe47c0-96cd-4379-9f15-8327e095b80e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:41.172279489Z\",\n      \"endTime\": \"2026-08-02T09:53:50.007487871Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:151]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019052c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wmrgqh8gzn96\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0fbe47c0-96cd-4379-9f15-8327e095b80e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0fbe47c0-96cd-4379-9f15-8327e095b80e\",\n      \"name\": \"0fbe47c0-96cd-4379-9f15-8327e095b80e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:41.172279489Z\",\n      \"endTime\": \"2026-08-02T09:53:50.007487871Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2a50e8acee6791bed11d0b84f99fecf3bc52e8a2aac36fcd0057e3bfa090aa96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"af673c3f3931c89c69892df1b525596bf2fb5688460fa909b7990682782a2796","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001475b00\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a3dde528-d0eb-458a-a5eb-c741c73cfea4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a3dde528-d0eb-458a-a5eb-c741c73cfea4\",\n      \"name\": \"a3dde528-d0eb-458a-a5eb-c741c73cfea4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:35:34.710183543Z\",\n      \"endTime\": \"2026-08-02T09:54:39.901296729Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001475b00\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a3dde528-d0eb-458a-a5eb-c741c73cfea4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a3dde528-d0eb-458a-a5eb-c741c73cfea4\",\n      \"name\": \"a3dde528-d0eb-458a-a5eb-c741c73cfea4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:35:34.710183543Z\",\n      \"endTime\": \"2026-08-02T09:54:39.901296729Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4108814e68729132d6fab582298128d9a1649dea1dabde4dab7871150d5e295e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"2e944f7dcda11dea00030f48665f964060e872901fa8be59db56f811abce6888","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:98]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000439100\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-schbwv7s4vqj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/433a98a2-2a48-4b1a-ad1a-2eb9672f77a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/433a98a2-2a48-4b1a-ad1a-2eb9672f77a2\",\n      \"name\": \"433a98a2-2a48-4b1a-ad1a-2eb9672f77a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:33:58.846876088Z\",\n      \"endTime\": \"2026-08-02T09:53:00.063648031Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:98]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000439100\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-schbwv7s4vqj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/433a98a2-2a48-4b1a-ad1a-2eb9672f77a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/433a98a2-2a48-4b1a-ad1a-2eb9672f77a2\",\n      \"name\": \"433a98a2-2a48-4b1a-ad1a-2eb9672f77a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:33:58.846876088Z\",\n      \"endTime\": \"2026-08-02T09:53:00.063648031Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"468831208b7496b85814a9591e6c7a27facd2ec51c087f934a90826b72d265f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"99f53693e4f66cf8cd29dad172acb0d2eb966d3db7b113aaa865daef312a6804","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-x5275t\" with version 4.20.27 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010888e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-x5275t: failed waiting for cluster=\"cluster-zstream-4-20-x5275t\" in resourcegroup=\"rg-zstream-upgrade-4-20-fs7v9z22qchl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2e46fd-96fc-4ba6-b840-436a57da57e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2e46fd-96fc-4ba6-b840-436a57da57e9\",\n      \"name\": \"6f2e46fd-96fc-4ba6-b840-436a57da57e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:20:39.561995336Z\",\n      \"endTime\": \"2026-08-02T10:39:42.899977522Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-x5275t\" with version 4.20.27 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010888e0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-x5275t: failed waiting for cluster=\"cluster-zstream-4-20-x5275t\" in resourcegroup=\"rg-zstream-upgrade-4-20-fs7v9z22qchl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2e46fd-96fc-4ba6-b840-436a57da57e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2e46fd-96fc-4ba6-b840-436a57da57e9\",\n      \"name\": \"6f2e46fd-96fc-4ba6-b840-436a57da57e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:20:39.561995336Z\",\n      \"endTime\": \"2026-08-02T10:39:42.899977522Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"58dbafc8b3f227ec0c0d69079acf858c50e0f9bd80c02a608524aa6a9bce483c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"052ebe2c3c97bb9a568990baa076d6e2109f823a9569c9d5902807103f1e7ba4","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-19-nhx2wp\" with version 4.19.39 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00097aa00\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-nhx2wp: failed waiting for cluster=\"cluster-zstream-4-19-nhx2wp\" in resourcegroup=\"rg-zstream-upgrade-4-19-rn966m5vc2n7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3263c87f-1b2a-4019-8a07-2574b3e4a978\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3263c87f-1b2a-4019-8a07-2574b3e4a978\",\n      \"name\": \"3263c87f-1b2a-4019-8a07-2574b3e4a978\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:20:38.264467464Z\",\n      \"endTime\": \"2026-08-02T10:39:42.477082291Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-19-nhx2wp\" with version 4.19.39 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00097aa00\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-nhx2wp: failed waiting for cluster=\"cluster-zstream-4-19-nhx2wp\" in resourcegroup=\"rg-zstream-upgrade-4-19-rn966m5vc2n7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3263c87f-1b2a-4019-8a07-2574b3e4a978\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3263c87f-1b2a-4019-8a07-2574b3e4a978\",\n      \"name\": \"3263c87f-1b2a-4019-8a07-2574b3e4a978\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:20:38.264467464Z\",\n      \"endTime\": \"2026-08-02T10:39:42.477082291Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5df0cdfe09005cf714b58072edb361989cf21fea820427e380183a758142bc0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"e04dbd31283ca9365e3d1599ffde4d002fd0443462663b99ce14916d32dabe89","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:149]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001412750\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-sf8fnq7ckvqz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sf8fnq7ckvqz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:149]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001412750\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-sf8fnq7ckvqz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sf8fnq7ckvqz, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-sf8fnq7ckvqz\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sf8fnq7ckvqz, error: context deadline exceeded"}},{"row_id":"7044bc0612c5f5e4c7ec34306b8f7dec8113866baf1ba0fa484c30af3a12c85d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"1bc26a0d8ae0ce7fe6f1aba0d8fc87d14bcd6a2b503d2c98efa3e089d66e2292","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:95]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000afc3c0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-4klq7d2pktl6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/99457e0f-0b14-423b-bb59-8608f2ea25b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/99457e0f-0b14-423b-bb59-8608f2ea25b8\",\n      \"name\": \"99457e0f-0b14-423b-bb59-8608f2ea25b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:03.731452984Z\",\n      \"endTime\": \"2026-08-02T09:53:20.117946772Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:95]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000afc3c0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-4klq7d2pktl6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/99457e0f-0b14-423b-bb59-8608f2ea25b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/99457e0f-0b14-423b-bb59-8608f2ea25b8\",\n      \"name\": \"99457e0f-0b14-423b-bb59-8608f2ea25b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:03.731452984Z\",\n      \"endTime\": \"2026-08-02T09:53:20.117946772Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"77fabcc86e247455e89b5edab5d122674ef0e849dabf25e49d67d6ea52b0667a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"fcf93f141a243945f67f44b14e73f777d9be669780336c2e9993260d931f7481","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd1360\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-zrbkqqp97x2s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89700ddf-4dea-44e4-88e8-5b32393432ad\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89700ddf-4dea-44e4-88e8-5b32393432ad\",\n      \"name\": \"89700ddf-4dea-44e4-88e8-5b32393432ad\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:57.46338231Z\",\n      \"endTime\": \"2026-08-02T09:54:10.022162917Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dd1360\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-zrbkqqp97x2s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89700ddf-4dea-44e4-88e8-5b32393432ad\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/89700ddf-4dea-44e4-88e8-5b32393432ad\",\n      \"name\": \"89700ddf-4dea-44e4-88e8-5b32393432ad\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:57.46338231Z\",\n      \"endTime\": \"2026-08-02T09:54:10.022162917Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e9748b590a19a20ca945cd9ba84b6623bd9a39a74231667437330e38b57d238","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"16c7003b3e534c2db0acd58bb74f730ca9c0be4f2219f61b2816b7c3c71ee0d3","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000946480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-b8tzdmst7fvv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d9310a88-789a-413f-94fe-bbd77de07f32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d9310a88-789a-413f-94fe-bbd77de07f32\",\n      \"name\": \"d9310a88-789a-413f-94fe-bbd77de07f32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:21.5378839Z\",\n      \"endTime\": \"2026-08-02T09:53:39.994858773Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000946480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-b8tzdmst7fvv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d9310a88-789a-413f-94fe-bbd77de07f32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d9310a88-789a-413f-94fe-bbd77de07f32\",\n      \"name\": \"d9310a88-789a-413f-94fe-bbd77de07f32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:21.5378839Z\",\n      \"endTime\": \"2026-08-02T09:53:39.994858773Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7ee0d06df5b1ecc3ff89fc43121bfb3d4f05eea2ebb9c9efb108e51d3e657555","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"b40a1fb2ac4a8cba9f1d30b8d5fea9aa9626d88c7b7d0cc04bc9194ba36da5bb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-2xrctt6gqvhc/cluster-candidate-4-22-jzgl67 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005109e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-jzgl67: failed waiting for cluster=\"cluster-candidate-4-22-jzgl67\" in resourcegroup=\"rg-candidate-4-22-2xrctt6gqvhc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0d09561-daaf-4fc0-ada1-2d89894ad25d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0d09561-daaf-4fc0-ada1-2d89894ad25d\",\n      \"name\": \"b0d09561-daaf-4fc0-ada1-2d89894ad25d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:19:58.698754411Z\",\n      \"endTime\": \"2026-08-02T10:39:02.899668713Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-2xrctt6gqvhc/cluster-candidate-4-22-jzgl67 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005109e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-jzgl67: failed waiting for cluster=\"cluster-candidate-4-22-jzgl67\" in resourcegroup=\"rg-candidate-4-22-2xrctt6gqvhc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0d09561-daaf-4fc0-ada1-2d89894ad25d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0d09561-daaf-4fc0-ada1-2d89894ad25d\",\n      \"name\": \"b0d09561-daaf-4fc0-ada1-2d89894ad25d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:19:58.698754411Z\",\n      \"endTime\": \"2026-08-02T10:39:02.899668713Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"816b7825d8a0cb847e21cbd3ad5d4fa7253b539e2df3239d2df45955bc3b5b7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"cb6eb456a0df8f447cc45568b17d4fc671353f4f3886840340943f1ee9cd92c1","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f88a80\u003e: \n    failed to create HCP cluster ea-listg8p2lr: failed waiting for cluster=\"ea-listg8p2lr\" in resourcegroup=\"ea-list-zpwzzw4hr4s4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f8f6fba3-8dc3-43e1-a8de-b639f8946093\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f8f6fba3-8dc3-43e1-a8de-b639f8946093\",\n      \"name\": \"f8f6fba3-8dc3-43e1-a8de-b639f8946093\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:22:17.745429228Z\",\n      \"endTime\": \"2026-08-02T10:41:22.679642027Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f88a80\u003e: \n    failed to create HCP cluster ea-listg8p2lr: failed waiting for cluster=\"ea-listg8p2lr\" in resourcegroup=\"ea-list-zpwzzw4hr4s4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f8f6fba3-8dc3-43e1-a8de-b639f8946093\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f8f6fba3-8dc3-43e1-a8de-b639f8946093\",\n      \"name\": \"f8f6fba3-8dc3-43e1-a8de-b639f8946093\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:22:17.745429228Z\",\n      \"endTime\": \"2026-08-02T10:41:22.679642027Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"88470ece986d66627ee91dc4656915ea81700f2f99dd722a6f6d41f4f7e40c6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"eef1f7d9c831ab92e3e3bd5d202952d2fed84be97e70dc93fbbbc52dfe7294da","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f58fa0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-nqr8kk5t28mq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9a6942f7-8b76-41f6-9814-53dcafaf62c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9a6942f7-8b76-41f6-9814-53dcafaf62c2\",\n      \"name\": \"9a6942f7-8b76-41f6-9814-53dcafaf62c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:01.532783895Z\",\n      \"endTime\": \"2026-08-02T10:40:02.630115585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f58fa0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-nqr8kk5t28mq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9a6942f7-8b76-41f6-9814-53dcafaf62c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9a6942f7-8b76-41f6-9814-53dcafaf62c2\",\n      \"name\": \"9a6942f7-8b76-41f6-9814-53dcafaf62c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:01.532783895Z\",\n      \"endTime\": \"2026-08-02T10:40:02.630115585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"926e3665481604c630c281fde64f7291ca9dc2a3572b3268dffc67b1301f33ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"1a30293f03d5bebc67ec881255dfa940c8186d7d08111f8bbe98b67241d9d72b","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00118f360\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-5shxkc7h96kd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f4e10e32-a5a2-48c8-bb8f-add91194d45d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f4e10e32-a5a2-48c8-bb8f-add91194d45d\",\n      \"name\": \"f4e10e32-a5a2-48c8-bb8f-add91194d45d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:57.74262599Z\",\n      \"endTime\": \"2026-08-02T09:54:00.011864472Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00118f360\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-5shxkc7h96kd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f4e10e32-a5a2-48c8-bb8f-add91194d45d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f4e10e32-a5a2-48c8-bb8f-add91194d45d\",\n      \"name\": \"f4e10e32-a5a2-48c8-bb8f-add91194d45d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:57.74262599Z\",\n      \"endTime\": \"2026-08-02T09:54:00.011864472Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"92cbb938bf40064f557b05876cd70bad5fb2dd20c8c8a8ce016b57bd73c6f33e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"ae05d6e06d1b0cf99d32c9111db86448d9f13d61f2c47276a1d08eefc967fc14","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-c4zqpd\" with install version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00069ce00\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-c4zqpd: failed waiting for cluster=\"cp-ystream-upgrade-4-21-c4zqpd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-b4dtm4754scw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ebb17e58-f37d-456d-8dda-e04ad5a89ffd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ebb17e58-f37d-456d-8dda-e04ad5a89ffd\",\n      \"name\": \"ebb17e58-f37d-456d-8dda-e04ad5a89ffd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:55.844645935Z\",\n      \"endTime\": \"2026-08-02T10:41:02.499727821Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-c4zqpd\" with install version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00069ce00\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-c4zqpd: failed waiting for cluster=\"cp-ystream-upgrade-4-21-c4zqpd\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-b4dtm4754scw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ebb17e58-f37d-456d-8dda-e04ad5a89ffd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ebb17e58-f37d-456d-8dda-e04ad5a89ffd\",\n      \"name\": \"ebb17e58-f37d-456d-8dda-e04ad5a89ffd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:55.844645935Z\",\n      \"endTime\": \"2026-08-02T10:41:02.499727821Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9a3d1e60f0276dfc1003ce9ac6e024280a17ac21f15e8950f082c548ddb12ecb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"05a2a50e8a6091e0075f3d39075b8f373088d19d53d30f09701a323b71da89a4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-thgbrpsnmsdd/cluster-candidate-4-21-vvlhtd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b94c00\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vvlhtd: failed waiting for cluster=\"cluster-candidate-4-21-vvlhtd\" in resourcegroup=\"rg-candidate-4-21-thgbrpsnmsdd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cfe40609-dfaa-4309-a817-b449ab573e51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cfe40609-dfaa-4309-a817-b449ab573e51\",\n      \"name\": \"cfe40609-dfaa-4309-a817-b449ab573e51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:40:52.659578391Z\",\n      \"endTime\": \"2026-08-02T10:00:01.276872508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-21-thgbrpsnmsdd/cluster-candidate-4-21-vvlhtd should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b94c00\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-vvlhtd: failed waiting for cluster=\"cluster-candidate-4-21-vvlhtd\" in resourcegroup=\"rg-candidate-4-21-thgbrpsnmsdd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cfe40609-dfaa-4309-a817-b449ab573e51\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cfe40609-dfaa-4309-a817-b449ab573e51\",\n      \"name\": \"cfe40609-dfaa-4309-a817-b449ab573e51\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:40:52.659578391Z\",\n      \"endTime\": \"2026-08-02T10:00:01.276872508Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9a854959beca5f83995354bf3a3984354a4d19f88757923d0f1ae1c7ff73dfdf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"38751658fb583189a0984e0db3eee96ce7a1813b6575d5270ee70cb61a4efdaa","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-7tjrwf with version 4.20.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009fc860\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-7tjrwf: failed waiting for cluster=\"np-version-upgrade-cluster-7tjrwf\" in resourcegroup=\"rg-np-version-upgrade-7tjrwf-gb7vxqn6kj8r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96cdc6-19a6-49b1-97a9-d42ab5c04593\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96cdc6-19a6-49b1-97a9-d42ab5c04593\",\n      \"name\": \"5f96cdc6-19a6-49b1-97a9-d42ab5c04593\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:25:02.326268128Z\",\n      \"endTime\": \"2026-08-02T10:44:02.561752064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-7tjrwf with version 4.20.32\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009fc860\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-7tjrwf: failed waiting for cluster=\"np-version-upgrade-cluster-7tjrwf\" in resourcegroup=\"rg-np-version-upgrade-7tjrwf-gb7vxqn6kj8r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96cdc6-19a6-49b1-97a9-d42ab5c04593\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5f96cdc6-19a6-49b1-97a9-d42ab5c04593\",\n      \"name\": \"5f96cdc6-19a6-49b1-97a9-d42ab5c04593\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:25:02.326268128Z\",\n      \"endTime\": \"2026-08-02T10:44:02.561752064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9ac99f288c1aaf09aa521ced950bf3368f82791f90657b57585d4e909dbb1d6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"c2862f8d2af906875d80375e9fa90f8ff9bc02e2357e85ccfe0589b8d49ed29e","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001bc3c0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-z7vrqxfs7z6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/96eaab45-e218-47ab-8385-511d2b8bea05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/96eaab45-e218-47ab-8385-511d2b8bea05\",\n      \"name\": \"96eaab45-e218-47ab-8385-511d2b8bea05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:01.062473396Z\",\n      \"endTime\": \"2026-08-02T09:53:20.146592724Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001bc3c0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-z7vrqxfs7z6k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/96eaab45-e218-47ab-8385-511d2b8bea05\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/96eaab45-e218-47ab-8385-511d2b8bea05\",\n      \"name\": \"96eaab45-e218-47ab-8385-511d2b8bea05\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:01.062473396Z\",\n      \"endTime\": \"2026-08-02T09:53:20.146592724Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aee24f9ea0ca4be7c2ddee159b87d777e42f989f5455b9aaba10d419cb52b011","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"758100d66a6991647c1f198297b30ee84054e8e323d4bb3a4dc793e5f36333c2","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e08ee0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-m6z62b4jc2tx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/37f499cd-df85-402b-81c7-871df8ababcf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/37f499cd-df85-402b-81c7-871df8ababcf\",\n      \"name\": \"37f499cd-df85-402b-81c7-871df8ababcf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:11.158027917Z\",\n      \"endTime\": \"2026-08-02T10:40:12.70290935Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e08ee0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-m6z62b4jc2tx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/37f499cd-df85-402b-81c7-871df8ababcf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/37f499cd-df85-402b-81c7-871df8ababcf\",\n      \"name\": \"37f499cd-df85-402b-81c7-871df8ababcf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:11.158027917Z\",\n      \"endTime\": \"2026-08-02T10:40:12.70290935Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b096c7a689e7ceffd00e4b1f8afaad14fab4003d3a51e5f0b9bd15da1d1e3561","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"3428e446bfa285f9bf1d24feb83c705e1ee307dcfb4a3915e67574e287931754","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-4c9krr with version 4.21.23\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d80ec0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-4c9krr: failed waiting for cluster=\"np-version-upgrade-cluster-4c9krr\" in resourcegroup=\"rg-np-version-upgrade-4c9krr-wrfbrlj4lbbr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/146bbd4a-743f-4661-9c79-1a8d6a5e97cd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/146bbd4a-743f-4661-9c79-1a8d6a5e97cd\",\n      \"name\": \"146bbd4a-743f-4661-9c79-1a8d6a5e97cd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:22:48.128797596Z\",\n      \"endTime\": \"2026-08-02T10:41:52.721915786Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-4c9krr with version 4.21.23\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d80ec0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-4c9krr: failed waiting for cluster=\"np-version-upgrade-cluster-4c9krr\" in resourcegroup=\"rg-np-version-upgrade-4c9krr-wrfbrlj4lbbr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/146bbd4a-743f-4661-9c79-1a8d6a5e97cd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/146bbd4a-743f-4661-9c79-1a8d6a5e97cd\",\n      \"name\": \"146bbd4a-743f-4661-9c79-1a8d6a5e97cd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:22:48.128797596Z\",\n      \"endTime\": \"2026-08-02T10:41:52.721915786Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b920992b0adfaedd55629611ef098ad805fc9b9c858e286700ae1fd9a2b81126","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"ab42de5f46497a16c405d24d5aa9cde06b8fea1cc967183d5966c2a2ab8f84ee","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000954860\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-qzhvx5bbqbk9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/016f96d1-550a-41f2-9fc1-b4db6bb899c9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/016f96d1-550a-41f2-9fc1-b4db6bb899c9\",\n      \"name\": \"016f96d1-550a-41f2-9fc1-b4db6bb899c9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:25.833585484Z\",\n      \"endTime\": \"2026-08-02T10:40:42.522944424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000954860\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-qzhvx5bbqbk9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/016f96d1-550a-41f2-9fc1-b4db6bb899c9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/016f96d1-550a-41f2-9fc1-b4db6bb899c9\",\n      \"name\": \"016f96d1-550a-41f2-9fc1-b4db6bb899c9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:25.833585484Z\",\n      \"endTime\": \"2026-08-02T10:40:42.522944424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bfb1eec56fbcc9876c1987d534c3a9a977e1aaf1f24b4ac0e7cf27f63cd67d80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"ea5855a74ab16b441b7b8667c71eb933594848719f04233ef7b8daee27e38120","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012d48e0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-fqw8f7c4wwz6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/24c83ba8-3e7d-4e9d-acd7-818541d03e5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/24c83ba8-3e7d-4e9d-acd7-818541d03e5b\",\n      \"name\": \"24c83ba8-3e7d-4e9d-acd7-818541d03e5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:22:16.371732004Z\",\n      \"endTime\": \"2026-08-02T10:41:22.659684191Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012d48e0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-fqw8f7c4wwz6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/24c83ba8-3e7d-4e9d-acd7-818541d03e5b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/24c83ba8-3e7d-4e9d-acd7-818541d03e5b\",\n      \"name\": \"24c83ba8-3e7d-4e9d-acd7-818541d03e5b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:22:16.371732004Z\",\n      \"endTime\": \"2026-08-02T10:41:22.659684191Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c57f9fc29cc35dbacba75ee57af6749483b9e781031d47266e2dc9c2d68c9b31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"8fa8b02b7d8b636eef0c10e97ce96c5b5d84f42b39fc1ad41b2f7c1c3750d7de","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-qm2jhm\" with version 4.22.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f14c40\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-qm2jhm: failed waiting for cluster=\"cluster-zstream-4-22-qm2jhm\" in resourcegroup=\"rg-zstream-upgrade-4-22-7prtcnrz9p7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/604d2e18-9f93-40ae-8c59-0ae91f105f7a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/604d2e18-9f93-40ae-8c59-0ae91f105f7a\",\n      \"name\": \"604d2e18-9f93-40ae-8c59-0ae91f105f7a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:20:29.396777978Z\",\n      \"endTime\": \"2026-08-02T10:39:42.921782454Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-qm2jhm\" with version 4.22.6 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f14c40\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-qm2jhm: failed waiting for cluster=\"cluster-zstream-4-22-qm2jhm\" in resourcegroup=\"rg-zstream-upgrade-4-22-7prtcnrz9p7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/604d2e18-9f93-40ae-8c59-0ae91f105f7a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/604d2e18-9f93-40ae-8c59-0ae91f105f7a\",\n      \"name\": \"604d2e18-9f93-40ae-8c59-0ae91f105f7a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:20:29.396777978Z\",\n      \"endTime\": \"2026-08-02T10:39:42.921782454Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c84937a68e421fcb3cd0ea021da430b32a834da17f8bbbdfec170f03fd235f4b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"403cf24078d9fc5ef87306cc4f3947de25eca13b005e2ccbead7322773601632","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-20-p79b4rkhtstx/cluster-candidate-4-20-qrm8jx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92d80\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-qrm8jx: failed waiting for cluster=\"cluster-candidate-4-20-qrm8jx\" in resourcegroup=\"rg-candidate-4-20-p79b4rkhtstx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8bade74f-a4fb-450a-a9c8-31a90fb5df6a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8bade74f-a4fb-450a-a9c8-31a90fb5df6a\",\n      \"name\": \"8bade74f-a4fb-450a-a9c8-31a90fb5df6a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:35:10.119552619Z\",\n      \"endTime\": \"2026-08-02T09:54:10.020290658Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-20-p79b4rkhtstx/cluster-candidate-4-20-qrm8jx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a92d80\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-qrm8jx: failed waiting for cluster=\"cluster-candidate-4-20-qrm8jx\" in resourcegroup=\"rg-candidate-4-20-p79b4rkhtstx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8bade74f-a4fb-450a-a9c8-31a90fb5df6a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8bade74f-a4fb-450a-a9c8-31a90fb5df6a\",\n      \"name\": \"8bade74f-a4fb-450a-a9c8-31a90fb5df6a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:35:10.119552619Z\",\n      \"endTime\": \"2026-08-02T09:54:10.020290658Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cd279756848c73180ef0a4afc0760279d4b4cdb4ebee17cfbdc2a7dd23b74c46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"c0a31610c203d33384a042a830ae08ce812df9f1fb4d5c64c588207e18c01528","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-wpfbzb\" with version 4.21.22 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eb0240\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-wpfbzb: failed waiting for cluster=\"cluster-zstream-4-21-wpfbzb\" in resourcegroup=\"rg-zstream-upgrade-4-21-hqwpx2rp9hwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5a86e2da-8342-4e05-9a7b-646fec5d1f57\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5a86e2da-8342-4e05-9a7b-646fec5d1f57\",\n      \"name\": \"5a86e2da-8342-4e05-9a7b-646fec5d1f57\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:20:45.199672861Z\",\n      \"endTime\": \"2026-08-02T10:39:52.85500071Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-wpfbzb\" with version 4.21.22 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000eb0240\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-wpfbzb: failed waiting for cluster=\"cluster-zstream-4-21-wpfbzb\" in resourcegroup=\"rg-zstream-upgrade-4-21-hqwpx2rp9hwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5a86e2da-8342-4e05-9a7b-646fec5d1f57\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5a86e2da-8342-4e05-9a7b-646fec5d1f57\",\n      \"name\": \"5a86e2da-8342-4e05-9a7b-646fec5d1f57\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:20:45.199672861Z\",\n      \"endTime\": \"2026-08-02T10:39:52.85500071Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d2dc3473e26a0c9d3f05f6a961faae679df7deb28301782ea16bca54a34ae608","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"051cf6ecf727851a70ff0cd34517780b1e9e358968115dffad4d43429bd5ffc8","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001a5360\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xjxxj2swjnz9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/df395857-64f2-4cba-a384-dba5dbda1151\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/df395857-64f2-4cba-a384-dba5dbda1151\",\n      \"name\": \"df395857-64f2-4cba-a384-dba5dbda1151\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:04.574656397Z\",\n      \"endTime\": \"2026-08-02T10:40:12.775068231Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0001a5360\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xjxxj2swjnz9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/df395857-64f2-4cba-a384-dba5dbda1151\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/df395857-64f2-4cba-a384-dba5dbda1151\",\n      \"name\": \"df395857-64f2-4cba-a384-dba5dbda1151\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:04.574656397Z\",\n      \"endTime\": \"2026-08-02T10:40:12.775068231Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d45a7c7ccf789074a0a6bb987a8b92f85c1990966c51c79e6613099abd5ec0b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"52038d3d19bf3139552e5ab7d7020bef9adf291bda0c9e9fd0263000496ac854","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004ae680\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-774f7qvh8zdv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2628e2-cfb7-4630-92d6-e86eb6db9b46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2628e2-cfb7-4630-92d6-e86eb6db9b46\",\n      \"name\": \"6f2628e2-cfb7-4630-92d6-e86eb6db9b46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:35.592173312Z\",\n      \"endTime\": \"2026-08-02T10:40:42.509887887Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004ae680\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-774f7qvh8zdv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2628e2-cfb7-4630-92d6-e86eb6db9b46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f2628e2-cfb7-4630-92d6-e86eb6db9b46\",\n      \"name\": \"6f2628e2-cfb7-4630-92d6-e86eb6db9b46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:35.592173312Z\",\n      \"endTime\": \"2026-08-02T10:40:42.509887887Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e03327dd27f74179ca4d6612076f0292f7dbfc01008e4c52c434a57a6bde1485","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"e1540453f45e8fd5f79082f8ef525ab44d396ab67e8eb0497e5dda49555ef44b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"a4aad4c43570be1a3f9d2d924b3f6f5e8247e1935408a40920e54c59ba43a538","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-t5gbrr with version 4.21.23\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062e4c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-t5gbrr: failed waiting for cluster=\"np-version-upgrade-cluster-t5gbrr\" in resourcegroup=\"rg-np-version-upgrade-t5gbrr-9m92wdf9smth\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a893a929-5de6-4c11-9274-877b11423a45\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a893a929-5de6-4c11-9274-877b11423a45\",\n      \"name\": \"a893a929-5de6-4c11-9274-877b11423a45\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:22:58.969255601Z\",\n      \"endTime\": \"2026-08-02T10:42:02.681869225Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-t5gbrr with version 4.21.23\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062e4c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-t5gbrr: failed waiting for cluster=\"np-version-upgrade-cluster-t5gbrr\" in resourcegroup=\"rg-np-version-upgrade-t5gbrr-9m92wdf9smth\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a893a929-5de6-4c11-9274-877b11423a45\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a893a929-5de6-4c11-9274-877b11423a45\",\n      \"name\": \"a893a929-5de6-4c11-9274-877b11423a45\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:22:58.969255601Z\",\n      \"endTime\": \"2026-08-02T10:42:02.681869225Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e6b4796f88504d9301436bb71d4efc9f12c02254e59419e2330e7d11e6b39cf6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"60f68294f0de255f611b75fe84b7d0107a776535c9c347d611014215381bca2b","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000906a20\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-n9jxzw2mh8nf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b2328641-a92c-449e-ae11-b4e1b37ed5a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b2328641-a92c-449e-ae11-b4e1b37ed5a2\",\n      \"name\": \"b2328641-a92c-449e-ae11-b4e1b37ed5a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:07.535948206Z\",\n      \"endTime\": \"2026-08-02T09:53:20.13427646Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000906a20\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-n9jxzw2mh8nf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b2328641-a92c-449e-ae11-b4e1b37ed5a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b2328641-a92c-449e-ae11-b4e1b37ed5a2\",\n      \"name\": \"b2328641-a92c-449e-ae11-b4e1b37ed5a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:34:07.535948206Z\",\n      \"endTime\": \"2026-08-02T09:53:20.13427646Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f52f453bb786c467f1268ccc12b26f0ab85fd1df365af48eaa2d25b18e0b12af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"c59f01db4e6f5f62a207e37ba88d75f58c2705406f4799ff08454e16bd714d9d","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aa8d00\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-xxd89b5gsd7p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d9000a4-f892-48f3-beba-a01dcf970841\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d9000a4-f892-48f3-beba-a01dcf970841\",\n      \"name\": \"6d9000a4-f892-48f3-beba-a01dcf970841\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:38:30.54644304Z\",\n      \"endTime\": \"2026-08-02T09:57:39.932107313Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aa8d00\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-xxd89b5gsd7p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d9000a4-f892-48f3-beba-a01dcf970841\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6d9000a4-f892-48f3-beba-a01dcf970841\",\n      \"name\": \"6d9000a4-f892-48f3-beba-a01dcf970841\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:38:30.54644304Z\",\n      \"endTime\": \"2026-08-02T09:57:39.932107313Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f85c8ce8de96c487c2909f2798e95cb0eafbd5fde48048dcd18638670ea191f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"6767756089139aab74b8a0e4afea6f3d529957e75af8bbec4c5d9f48607a5b54","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f569c0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-tpg49l4jbx5d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/565715c5-d553-4e22-9e42-0fc997a2a44c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/565715c5-d553-4e22-9e42-0fc997a2a44c\",\n      \"name\": \"565715c5-d553-4e22-9e42-0fc997a2a44c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:22.100482516Z\",\n      \"endTime\": \"2026-08-02T10:40:22.835720238Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f569c0\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-tpg49l4jbx5d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/565715c5-d553-4e22-9e42-0fc997a2a44c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/565715c5-d553-4e22-9e42-0fc997a2a44c\",\n      \"name\": \"565715c5-d553-4e22-9e42-0fc997a2a44c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T10:21:22.100482516Z\",\n      \"endTime\": \"2026-08-02T10:40:22.835720238Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fb67cd1062a919020072cae1edcee29e7ae809853596cbf8dda1b98fa01dda24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2083844169058488320","occurred_at":"2026-08-02T09:15:40Z","failed_at":"e2e","signature_id":"6d309d366f045b03e648f62a1615aad3e21f681888bbee67c72198bd0dd762f2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:105]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0003fe7c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-q7k5fm2cgm92\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66774fff-682b-4c22-af08-5f9dea95118b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66774fff-682b-4c22-af08-5f9dea95118b\",\n      \"name\": \"66774fff-682b-4c22-af08-5f9dea95118b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:36:47.470711734Z\",\n      \"endTime\": \"2026-08-02T09:55:49.913729058Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:105]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0003fe7c0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-q7k5fm2cgm92\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66774fff-682b-4c22-af08-5f9dea95118b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66774fff-682b-4c22-af08-5f9dea95118b\",\n      \"name\": \"66774fff-682b-4c22-af08-5f9dea95118b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T09:36:47.470711734Z\",\n      \"endTime\": \"2026-08-02T09:55:49.913729058Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":38,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b"],"matched_occurrences":38,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2083734553368727552","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-02T02:00:06Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3228543f6e14859388d9ed96d6d0216d03d3678090b3698dd89fe5d1133246e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2083734553368727552","occurred_at":"2026-08-02T02:00:06Z","failed_at":"e2e","signature_id":"d45338114907e1f99cd34e1b097973074d907ac30b280168e474872bc1831e78","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00214e018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-l54jll2tvrwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0bdec85c-5f48-447c-b8b0-4dedc7990652\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0bdec85c-5f48-447c-b8b0-4dedc7990652\",\n      \"name\": \"0bdec85c-5f48-447c-b8b0-4dedc7990652\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T02:20:43.022860317Z\",\n      \"endTime\": \"2026-08-02T02:39:48.638662525Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00214e018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-l54jll2tvrwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0bdec85c-5f48-447c-b8b0-4dedc7990652\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0bdec85c-5f48-447c-b8b0-4dedc7990652\",\n      \"name\": \"0bdec85c-5f48-447c-b8b0-4dedc7990652\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-02T02:20:43.022860317Z\",\n      \"endTime\": \"2026-08-02T02:39:48.638662525Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}