{"meta":{"date":"2026-08-13","timezone":"UTC","generated_at":"2026-08-26T02:25:33Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":9,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088038044278984704","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-13T23:00:38Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"53c0fbee1995ce0f1fcd1a70fcd03597f112e69484ac72eb317a9429347186fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088038044278984704","occurred_at":"2026-08-13T23:00:38Z","failed_at":"e2e","signature_id":"6b8fcd2703f57320f816bd4e0c92f70acab07d54b540ea5a0f114864c8d8bc88","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n \u003c*errors.joinError | 0xc002918a50\u003e: \n VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zpk2h-89fng_OSDisk for VM des-encrypt-des-np-zpk2h-89fng has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n disk des-encrypt-des-np-zpk2h-f5zfj_OSDisk for VM des-encrypt-des-np-zpk2h-f5zfj has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n \u003c*errors.joinError | 0xc002918a50\u003e: \n VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zpk2h-89fng_OSDisk for VM des-encrypt-des-np-zpk2h-89fng has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n disk des-encrypt-des-np-zpk2h-f5zfj_OSDisk for VM des-encrypt-des-np-zpk2h-f5zfj has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"46ba20bd0022bbab2077ca36c67330fccd8ca0996c8be8400f5fdb8a593b354d","failure_pattern":"VerifyVMOSDiskCustomerEncryption(nodePool=\u003cnodepool\u003e) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zpk2h-89fng_OSDisk for VM des-encrypt-des-np-zpk2h-89fng has encryption type EncryptionAtRestWithPlatformKey, expected","search_query":"VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zpk2h-89fng_OSDisk for VM des-encrypt-des-np-zpk2h-89fng has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey"}},{"row_id":"9db3b959c23ecd76124a13d1b33a776c7b10933eb3b48413ebf5b3c51910176f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088038044278984704","occurred_at":"2026-08-13T23:00:38Z","failed_at":"e2e","signature_id":"a6dc99ecaaca936b7dec45c6efa92ed7510efcec862beaed09ca307257ca6804","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-22-gc96gfnnfh7f/cluster-nightly-4-22-hjgzzn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d44860\u003e: \n failed to create HCP cluster cluster-nightly-4-22-hjgzzn: failed waiting for cluster=\"cluster-nightly-4-22-hjgzzn\" in resourcegroup=\"rg-nightly-4-22-gc96gfnnfh7f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b8781edb-7125-44c1-89c9-ab2300095ebb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b8781edb-7125-44c1-89c9-ab2300095ebb\",\n \"name\": \"b8781edb-7125-44c1-89c9-ab2300095ebb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T23:32:10.892570447Z\",\n \"endTime\": \"2026-08-13T23:51:11.867428218Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-4-22-gc96gfnnfh7f/cluster-nightly-4-22-hjgzzn should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d44860\u003e: \n failed to create HCP cluster cluster-nightly-4-22-hjgzzn: failed waiting for cluster=\"cluster-nightly-4-22-hjgzzn\" in resourcegroup=\"rg-nightly-4-22-gc96gfnnfh7f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b8781edb-7125-44c1-89c9-ab2300095ebb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b8781edb-7125-44c1-89c9-ab2300095ebb\",\n \"name\": \"b8781edb-7125-44c1-89c9-ab2300095ebb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T23:32:10.892570447Z\",\n \"endTime\": \"2026-08-13T23:51:11.867428218Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cb7765b211614231a455b4217d8ccdf353c7be3989f76e71b6db442dd1be64e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088038044278984704","occurred_at":"2026-08-13T23:00:38Z","failed_at":"e2e","signature_id":"50d20647dc4e343420ed4311b1653506de8b52adcdc8d7e9904de0aebda52739","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-ps74xj8djqv9/cluster-nightly-5-1-jxrdzr should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0018b6570\u003e: \n failed to create HCP cluster cluster-nightly-5-1-jxrdzr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: failed waiting for cluster=\"cluster-nightly-5-1-jxrdzr\" in resourcegroup=\"rg-nightly-5-1-ps74xj8djqv9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-ps74xj8djqv9/cluster-nightly-5-1-jxrdzr should provision\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0018b6570\u003e: \n failed to create HCP cluster cluster-nightly-5-1-jxrdzr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: failed waiting for cluster=\"cluster-nightly-5-1-jxrdzr\" in resourcegroup=\"rg-nightly-5-1-ps74xj8djqv9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-jxrdzr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: failed waiting for cluster=\"cluster-nightly-5-1-jxrdzr\" in resourcegroup=\"rg-nightly-5-1-ps74xj8djqv9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: context deadline exceeded"}},{"row_id":"d12560b86a18b2283ed743e3ed813500e9a00fcb17ef975008324b85fb1fb4f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2088038044278984704","occurred_at":"2026-08-13T23:00:38Z","failed_at":"e2e","signature_id":"7c048170bd9fa51a7ee93f4d46bf9fc646bd0775fe746ada1f6fef606b9ae017","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001489350\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-z2tkbzk2p5xk, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-z2tkbzk2p5xk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-z2tkbzk2p5xk, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001489350\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-z2tkbzk2p5xk, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-z2tkbzk2p5xk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-z2tkbzk2p5xk, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-jxrdzr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: failed waiting for cluster=\"cluster-nightly-5-1-jxrdzr\" in resourcegroup=\"rg-nightly-5-1-ps74xj8djqv9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","46ba20bd0022bbab2077ca36c67330fccd8ca0996c8be8400f5fdb8a593b354d","d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088019536656732160","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-13T21:47:05Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"54393718cb0c2d944bda8538bff91a33a0a570abda97e0e3058e7f60bec21f44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088019536656732160","occurred_at":"2026-08-13T21:47:05Z","failed_at":"e2e","signature_id":"9f090ed1e62064ca490c207349dc600d3394d354f5c7e415efb839f8bb9c1c80","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d8e6c0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rkc5mbl4jhhf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\",\n \"name\": \"56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T22:51:09.443199677Z\",\n \"endTime\": \"2026-08-13T23:10:11.643927Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d8e6c0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rkc5mbl4jhhf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\",\n \"name\": \"56d1eae1-1c5b-4a4d-be74-e342e7fb8f62\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T22:51:09.443199677Z\",\n \"endTime\": \"2026-08-13T23:10:11.643927Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"de18610d4b13bfb9537c4d76612fe78c9c4b8672f5709bd744d17ad08cc9e633","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2088019536656732160","occurred_at":"2026-08-13T21:47:05Z","failed_at":"e2e","signature_id":"a2e879187cb6972854c4075c1f2c445ce736f5032aa71780d6b866151a01a179","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110a340\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-l22tf5tl8zxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/d3ea7256-0b78-469e-a13e-56a00e17bf30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/d3ea7256-0b78-469e-a13e-56a00e17bf30\",\n \"name\": \"d3ea7256-0b78-469e-a13e-56a00e17bf30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T22:13:33.517913078Z\",\n \"endTime\": \"2026-08-13T22:32:39.987880545Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110a340\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-l22tf5tl8zxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/d3ea7256-0b78-469e-a13e-56a00e17bf30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/d3ea7256-0b78-469e-a13e-56a00e17bf30\",\n \"name\": \"d3ea7256-0b78-469e-a13e-56a00e17bf30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T22:13:33.517913078Z\",\n \"endTime\": \"2026-08-13T22:32:39.987880545Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-13T16:29:44Z"},"failed_at":["e2e"],"failed_test_count":15,"occurrences":[{"row_id":"1bc0aac258678eeda6d0965c5c72c4106e92c5371887da55aa81fcbb7523ab77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"9f6f42bed68e6d3b86d81931d02f06532f74fc43be808bc0172db5fcca80bec4","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"1e48816127d2206eb0e1a570d5acfeee1eff8e19fd8c684e6f891e642b00b2b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"fae07ddb718bf98f6d3d1edced5cb60ae56e409927f36cedc96217c0cd7a290c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-fl5dzt9shx72/cluster-candidate-5-0-fsc67s should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000644380\u003e: \n failed to create HCP cluster cluster-candidate-5-0-fsc67s: failed waiting for cluster=\"cluster-candidate-5-0-fsc67s\" in resourcegroup=\"rg-candidate-5-0-fl5dzt9shx72\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f29b4185-b022-4212-8a83-448e94357420\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f29b4185-b022-4212-8a83-448e94357420\",\n \"name\": \"f29b4185-b022-4212-8a83-448e94357420\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T17:31:47.763208934Z\",\n \"endTime\": \"2026-08-13T17:50:48.076190757Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-fl5dzt9shx72/cluster-candidate-5-0-fsc67s should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000644380\u003e: \n failed to create HCP cluster cluster-candidate-5-0-fsc67s: failed waiting for cluster=\"cluster-candidate-5-0-fsc67s\" in resourcegroup=\"rg-candidate-5-0-fl5dzt9shx72\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f29b4185-b022-4212-8a83-448e94357420\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f29b4185-b022-4212-8a83-448e94357420\",\n \"name\": \"f29b4185-b022-4212-8a83-448e94357420\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T17:31:47.763208934Z\",\n \"endTime\": \"2026-08-13T17:50:48.076190757Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9509a4930132704b80afcf0f9ab5f4573c4e906011498304e9125ac11a4a978","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [ignition-server-proxy, kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"253badbcbd0776be0add6156d94d490ab076116015e8a257eee5e1b0f9f39368","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"a9e21576b8c2932aef14ea318a2a302475569455a7e3c670e4e4b6c84c04d9f2","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011dc440\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-z7twstkk877q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/ae54c7de-b8d4-4041-9176-22b9c96c29a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/ae54c7de-b8d4-4041-9176-22b9c96c29a5\",\n \"name\": \"ae54c7de-b8d4-4041-9176-22b9c96c29a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:58:22.806879081Z\",\n \"endTime\": \"2026-08-13T17:17:27.029777623Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.y5u5u9r0s4y5z1m.bj4u.westeurope.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011dc440\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-z7twstkk877q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/ae54c7de-b8d4-4041-9176-22b9c96c29a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/ae54c7de-b8d4-4041-9176-22b9c96c29a5\",\n \"name\": \"ae54c7de-b8d4-4041-9176-22b9c96c29a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:58:22.806879081Z\",\n \"endTime\": \"2026-08-13T17:17:27.029777623Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \\\"https://api-int.y5u5u9r0s4y5z1m.bj4u.westeurope.aroapp-hcp.io:443/healthz\\\": EOF; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, control-plane-pki-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, machine-approver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aeb99d9b1ad881fcb99495f400b224fcad0bc6f5c9ff6c3ffad9218a1eb1ffa7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: Get \"\u003curl\u003e\": EOF; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2ad8e56e1fd72e74ead355b828899c55447c4b4b34a1581da8a903997883e070","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"e36c1f212cf181bdf7c7a3efef0227f22922a029910af135e4fa995832869820","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-q8wb29\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90b40\u003e: \n failed to create HCP cluster np-noedge-q8wb29: failed waiting for cluster=\"np-noedge-q8wb29\" in resourcegroup=\"rg-np-noedge-q8wb29-scqh47gqfgtk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/49e390d3-8264-4961-b591-2ba8a2528384\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/49e390d3-8264-4961-b591-2ba8a2528384\",\n \"name\": \"49e390d3-8264-4961-b591-2ba8a2528384\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T18:04:01.864548732Z\",\n \"endTime\": \"2026-08-13T18:23:09.31047489Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-q8wb29\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90b40\u003e: \n failed to create HCP cluster np-noedge-q8wb29: failed waiting for cluster=\"np-noedge-q8wb29\" in resourcegroup=\"rg-np-noedge-q8wb29-scqh47gqfgtk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/49e390d3-8264-4961-b591-2ba8a2528384\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/49e390d3-8264-4961-b591-2ba8a2528384\",\n \"name\": \"49e390d3-8264-4961-b591-2ba8a2528384\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T18:04:01.864548732Z\",\n \"endTime\": \"2026-08-13T18:23:09.31047489Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: ingress-operator, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa4e3f3fddf55a14f372dba46754c68c0e6985edc043acb16ad92a33f436f738","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-network-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"342db9f3ca89045d04c95c8c795a643d28ca913f943f2723cbb25abeaa36859b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"5ef680a9f58e5968f085acafa4181a8cca9c772497009d797b9cbc55216675fd","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000540280\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-5mdmhgbzh58p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/af8ee527-2640-4dcb-92f9-765f4b0064ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/af8ee527-2640-4dcb-92f9-765f4b0064ce\",\n \"name\": \"af8ee527-2640-4dcb-92f9-765f4b0064ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:56:53.856645111Z\",\n \"endTime\": \"2026-08-13T17:15:56.849316189Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000540280\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-5mdmhgbzh58p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/af8ee527-2640-4dcb-92f9-765f4b0064ce\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/af8ee527-2640-4dcb-92f9-765f4b0064ce\",\n \"name\": \"af8ee527-2640-4dcb-92f9-765f4b0064ce\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:56:53.856645111Z\",\n \"endTime\": \"2026-08-13T17:15:56.849316189Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3ddad1876eb6c7a89f4e66a0a005cbc3b2eabbd9bb2f2bd0bf4591599e2fd873","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"3470f6e6551778c521fae7611d06158bd7b0f2d9f0f18172ea64158c3fb3b4bd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-vzgnsfg29j2j/cluster-candidate-4-22-5x9pt4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008be700\u003e: \n failed to create HCP cluster cluster-candidate-4-22-5x9pt4: failed waiting for cluster=\"cluster-candidate-4-22-5x9pt4\" in resourcegroup=\"rg-candidate-4-22-vzgnsfg29j2j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8131ef38-f0b6-49e5-94b9-386607403e5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8131ef38-f0b6-49e5-94b9-386607403e5f\",\n \"name\": \"8131ef38-f0b6-49e5-94b9-386607403e5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T17:31:43.935377005Z\",\n \"endTime\": \"2026-08-13T17:50:48.106242949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-vzgnsfg29j2j/cluster-candidate-4-22-5x9pt4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008be700\u003e: \n failed to create HCP cluster cluster-candidate-4-22-5x9pt4: failed waiting for cluster=\"cluster-candidate-4-22-5x9pt4\" in resourcegroup=\"rg-candidate-4-22-vzgnsfg29j2j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8131ef38-f0b6-49e5-94b9-386607403e5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8131ef38-f0b6-49e5-94b9-386607403e5f\",\n \"name\": \"8131ef38-f0b6-49e5-94b9-386607403e5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T17:31:43.935377005Z\",\n \"endTime\": \"2026-08-13T17:50:48.106242949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4aeb2c22e1846934277a7bf8e31488ed1933cd4f319643c6d464c6b97674ce48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"4e90b9d998f42658ba6457f2b76799a9f7e58cce249d22bbff156a35e6438459","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-rnm2tvm2mrmp/cluster-candidate-4-20-qwtx8f should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c08b00\u003e: \n failed to create HCP cluster cluster-candidate-4-20-qwtx8f: failed waiting for cluster=\"cluster-candidate-4-20-qwtx8f\" in resourcegroup=\"rg-candidate-4-20-rnm2tvm2mrmp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91167263-7a29-4fa4-99d6-8166e3b4d43c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91167263-7a29-4fa4-99d6-8166e3b4d43c\",\n \"name\": \"91167263-7a29-4fa4-99d6-8166e3b4d43c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T17:30:23.962927203Z\",\n \"endTime\": \"2026-08-13T17:49:28.261598664Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-rnm2tvm2mrmp/cluster-candidate-4-20-qwtx8f should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c08b00\u003e: \n failed to create HCP cluster cluster-candidate-4-20-qwtx8f: failed waiting for cluster=\"cluster-candidate-4-20-qwtx8f\" in resourcegroup=\"rg-candidate-4-20-rnm2tvm2mrmp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91167263-7a29-4fa4-99d6-8166e3b4d43c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91167263-7a29-4fa4-99d6-8166e3b4d43c\",\n \"name\": \"91167263-7a29-4fa4-99d6-8166e3b4d43c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T17:30:23.962927203Z\",\n \"endTime\": \"2026-08-13T17:49:28.261598664Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"62e371ce7f6f54c42abe40878a3b8f7ff7cda6031c2725220e595da31680bcbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"8b4e2592abe66dc05bc9ad4efbfe9bd369bb2c98a5c616f5afbb52dbbee7f2f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"1f8d2eaa98ff25669a7eeb95b723a4de8d2fa8764a770525baafc46df1a53cd9","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b94420\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-qbkwwv7bvchw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3e0badea-7de1-4906-94f7-05bdf32932c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3e0badea-7de1-4906-94f7-05bdf32932c7\",\n \"name\": \"3e0badea-7de1-4906-94f7-05bdf32932c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:57:00.949272396Z\",\n \"endTime\": \"2026-08-13T17:16:06.930089665Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b94420\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-qbkwwv7bvchw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3e0badea-7de1-4906-94f7-05bdf32932c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3e0badea-7de1-4906-94f7-05bdf32932c7\",\n \"name\": \"3e0badea-7de1-4906-94f7-05bdf32932c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:57:00.949272396Z\",\n \"endTime\": \"2026-08-13T17:16:06.930089665Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9c7a4c969d25fd861e3adde749c5165aad348ed6d752417b0579bb21378a7ef1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"4662f084d88a96bedec02db1e5cc94bd0b5ab04413acfc35afcac56c685d173e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060eba0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-kq8jngdkzqcj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f6db197c-dd59-47eb-856f-e158628e5faf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f6db197c-dd59-47eb-856f-e158628e5faf\",\n \"name\": \"f6db197c-dd59-47eb-856f-e158628e5faf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T17:37:52.956629852Z\",\n \"endTime\": \"2026-08-13T17:56:59.221357436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060eba0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-kq8jngdkzqcj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f6db197c-dd59-47eb-856f-e158628e5faf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f6db197c-dd59-47eb-856f-e158628e5faf\",\n \"name\": \"f6db197c-dd59-47eb-856f-e158628e5faf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T17:37:52.956629852Z\",\n \"endTime\": \"2026-08-13T17:56:59.221357436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a3fc8dc139a8a65f4e413d1ddaf03522399660eeca3b3a38154fc5a08ffa81b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"66ef78e183930b4044539e132fae6fd77f008b6b1e5c0e9142bb5f2c6f1f3265","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8f660\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-x6t6kwx6bp42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/65d119df-660f-4e58-9993-5f6118c5be2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/65d119df-660f-4e58-9993-5f6118c5be2d\",\n \"name\": \"65d119df-660f-4e58-9993-5f6118c5be2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:57:36.200799801Z\",\n \"endTime\": \"2026-08-13T17:16:36.805943703Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8f660\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-x6t6kwx6bp42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/65d119df-660f-4e58-9993-5f6118c5be2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/65d119df-660f-4e58-9993-5f6118c5be2d\",\n \"name\": \"65d119df-660f-4e58-9993-5f6118c5be2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:57:36.200799801Z\",\n \"endTime\": \"2026-08-13T17:16:36.805943703Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a21c55609829272b5f173dd8ce8e25d8c4d9a8eec84afefcdb262ef97c91acb7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [hosted-cluster-config-operator, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b199e19c1ca1e80188a8630ca0bb6a4622095aa49d35d85251cb9836007722a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"1e8a5190c6b519e6bcdb5cd13427d10302182e3754b2ed5028646e0a7266d796","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880340\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-8r9tpgrklzkp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/07101123-d638-438a-a217-7468c9aedf3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/07101123-d638-438a-a217-7468c9aedf3f\",\n \"name\": \"07101123-d638-438a-a217-7468c9aedf3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:56:50.524067582Z\",\n \"endTime\": \"2026-08-13T17:15:56.988850244Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880340\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-8r9tpgrklzkp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/07101123-d638-438a-a217-7468c9aedf3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/07101123-d638-438a-a217-7468c9aedf3f\",\n \"name\": \"07101123-d638-438a-a217-7468c9aedf3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:56:50.524067582Z\",\n \"endTime\": \"2026-08-13T17:15:56.988850244Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c7245dcf6ce5617d996bf7aeabe84302e2ba8a68bd6b65fe08787cc3a3d56f61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"50350edc4c7d5f835e6689a0d4e14af05941274d2afe9dfd3f5ccaac276423bf","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-mgrpt82ln4bh/cluster-candidate-4-21-cvgp9r should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010de6c0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-cvgp9r: failed waiting for cluster=\"cluster-candidate-4-21-cvgp9r\" in resourcegroup=\"rg-candidate-4-21-mgrpt82ln4bh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\",\n \"name\": \"c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T17:30:22.500953939Z\",\n \"endTime\": \"2026-08-13T17:49:28.209822365Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-mgrpt82ln4bh/cluster-candidate-4-21-cvgp9r should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010de6c0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-cvgp9r: failed waiting for cluster=\"cluster-candidate-4-21-cvgp9r\" in resourcegroup=\"rg-candidate-4-21-mgrpt82ln4bh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\",\n \"name\": \"c11e3bda-7b18-4cf4-9d5e-bc8c82684e7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T17:30:22.500953939Z\",\n \"endTime\": \"2026-08-13T17:49:28.209822365Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb8ca43a79863985fe5fa47ef1bbe80d7bc25200d089461928007bb1c0bcd82f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c9ab50b41bfa21321aaffc6013f024dc30c32991fb0e8bb5358b31b33e46f9cc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"ba7494a6c8814df7c2bba6f4ada3833c130e86360b32c05f3ce3226d149c0de1","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001027020\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-s6pxrqlhs4fd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/363acc2d-8995-4b0b-8042-e788a23c46c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/363acc2d-8995-4b0b-8042-e788a23c46c8\",\n \"name\": \"363acc2d-8995-4b0b-8042-e788a23c46c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:56:49.931217747Z\",\n \"endTime\": \"2026-08-13T17:15:56.947670699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001027020\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-s6pxrqlhs4fd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/363acc2d-8995-4b0b-8042-e788a23c46c8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/363acc2d-8995-4b0b-8042-e788a23c46c8\",\n \"name\": \"363acc2d-8995-4b0b-8042-e788a23c46c8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:56:49.931217747Z\",\n \"endTime\": \"2026-08-13T17:15:56.947670699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb8ca43a79863985fe5fa47ef1bbe80d7bc25200d089461928007bb1c0bcd82f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"df8358826596dc96a3b57bdffe491b9ca7d61b5b0b9065be0a2c658d8f9e5e99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087939673195286528","occurred_at":"2026-08-13T16:29:44Z","failed_at":"e2e","signature_id":"e815ffbc8459f58bfe52e56bcd82851f27a02fcad4d28992925670cbef6453f5","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001362c80\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m4w85gmwnx2g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b4037bc6-8c3f-4270-affb-68a1af331f3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b4037bc6-8c3f-4270-affb-68a1af331f3f\",\n \"name\": \"b4037bc6-8c3f-4270-affb-68a1af331f3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:58:24.22544747Z\",\n \"endTime\": \"2026-08-13T17:17:27.035958879Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, packageserver, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, olm-operator, openshift-controller-manager, catalog-operator, cluster-storage-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001362c80\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m4w85gmwnx2g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b4037bc6-8c3f-4270-affb-68a1af331f3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b4037bc6-8c3f-4270-affb-68a1af331f3f\",\n \"name\": \"b4037bc6-8c3f-4270-affb-68a1af331f3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T16:58:24.22544747Z\",\n \"endTime\": \"2026-08-13T17:17:27.035958879Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: dns-operator, packageserver, cluster-version-operator, openshift-route-controller-manager, cluster-network-operator, ingress-operator, csi-snapshot-controller-operator, olm-operator, openshift-controller-manager, catalog-operator, cluster-storage-operator, hosted-cluster-config-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-controller-manager deployment has 1 unavailable replicas, openshift-route-controller-manager deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18e7e3bddd3f5a1d71bd0a4e399cd7bd2e6186d31ff0833685a14cba11311d1d","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","18e7e3bddd3f5a1d71bd0a4e399cd7bd2e6186d31ff0833685a14cba11311d1d","26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","a21c55609829272b5f173dd8ce8e25d8c4d9a8eec84afefcdb262ef97c91acb7","aeb99d9b1ad881fcb99495f400b224fcad0bc6f5c9ff6c3ffad9218a1eb1ffa7","cb8ca43a79863985fe5fa47ef1bbe80d7bc25200d089461928007bb1c0bcd82f","d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","d9509a4930132704b80afcf0f9ab5f4573c4e906011498304e9125ac11a4a978","f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","fa4e3f3fddf55a14f372dba46754c68c0e6985edc043acb16ad92a33f436f738"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-13T12:05:32Z"},"failed_at":["e2e"],"failed_test_count":47,"occurrences":[{"row_id":"0ce278ed8b3daf3adb3d22f1ad6a94310d32e0ab1c4b907d47d08560cb84ca08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"3c613e199a0bf63f2287947ce3d7d2f24dc1274357038e57b57c080830c2ca68","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d06260\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ppvlxzngh4lp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/71e91ed4-1560-4521-b955-5af6b5afba6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/71e91ed4-1560-4521-b955-5af6b5afba6e\",\n \"name\": \"71e91ed4-1560-4521-b955-5af6b5afba6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:10.490311709Z\",\n \"endTime\": \"2026-08-13T12:54:25.014444464Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:106]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d06260\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ppvlxzngh4lp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/71e91ed4-1560-4521-b955-5af6b5afba6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/71e91ed4-1560-4521-b955-5af6b5afba6e\",\n \"name\": \"71e91ed4-1560-4521-b955-5af6b5afba6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:10.490311709Z\",\n \"endTime\": \"2026-08-13T12:54:25.014444464Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0d03166c970a17ab5553cac8f67433bfcc968e1f63936605e38a7e78abfb2919","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"defba06caca5b48f794a13c44769520226a187e375516a67a01a20d96e65c3c5","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a834a0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-gqtwmn4w9gsm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/6edfb8d0-e97a-4c8e-b7c8-d05f094b1d60\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/6edfb8d0-e97a-4c8e-b7c8-d05f094b1d60\",\n \"name\": \"6edfb8d0-e97a-4c8e-b7c8-d05f094b1d60\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:48.355423491Z\",\n \"endTime\": \"2026-08-13T13:45:57.081686601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a834a0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-gqtwmn4w9gsm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/6edfb8d0-e97a-4c8e-b7c8-d05f094b1d60\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/6edfb8d0-e97a-4c8e-b7c8-d05f094b1d60\",\n \"name\": \"6edfb8d0-e97a-4c8e-b7c8-d05f094b1d60\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:48.355423491Z\",\n \"endTime\": \"2026-08-13T13:45:57.081686601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1c022d24a3b5067db2368b7f46d96a486451f8ab59f0f751b6569ed4aaa96ba7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"f81fa22284e45a6764e5463480983eb8429545ebc5e73f81fde8cd24f7e51e5c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-7d849b\" with version 4.21.27 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009270c0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-7d849b: failed waiting for cluster=\"cluster-zstream-4-21-7d849b\" in resourcegroup=\"rg-zstream-upgrade-4-21-6jb88cv8fcrx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/bf4eeb6d-5c5d-476f-adb7-791f8277f7ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/bf4eeb6d-5c5d-476f-adb7-791f8277f7ab\",\n \"name\": \"bf4eeb6d-5c5d-476f-adb7-791f8277f7ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:49.26271235Z\",\n \"endTime\": \"2026-08-13T13:45:57.083397893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-7d849b\" with version 4.21.27 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009270c0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-7d849b: failed waiting for cluster=\"cluster-zstream-4-21-7d849b\" in resourcegroup=\"rg-zstream-upgrade-4-21-6jb88cv8fcrx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/bf4eeb6d-5c5d-476f-adb7-791f8277f7ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/bf4eeb6d-5c5d-476f-adb7-791f8277f7ab\",\n \"name\": \"bf4eeb6d-5c5d-476f-adb7-791f8277f7ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:49.26271235Z\",\n \"endTime\": \"2026-08-13T13:45:57.083397893Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1efd210d3542ff8d76915b1e71f068a3f1092a5041f7436be2cf18c3b7fd5f17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"efe21dc4f1633db367f4a633d5271810146f574c23abad6499cf2bb043236415","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a7b3e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-9qjdk2ssrxdm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2a05c823-bbc5-4a57-a75e-12d5b60cd167\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2a05c823-bbc5-4a57-a75e-12d5b60cd167\",\n \"name\": \"2a05c823-bbc5-4a57-a75e-12d5b60cd167\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:47.96714361Z\",\n \"endTime\": \"2026-08-13T12:54:55.022053173Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a7b3e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-9qjdk2ssrxdm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2a05c823-bbc5-4a57-a75e-12d5b60cd167\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2a05c823-bbc5-4a57-a75e-12d5b60cd167\",\n \"name\": \"2a05c823-bbc5-4a57-a75e-12d5b60cd167\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:47.96714361Z\",\n \"endTime\": \"2026-08-13T12:54:55.022053173Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"202c39e50d23a0c062269f196704fdfc0acffd8167fd8392129a34c9e6cec097","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"80b27ff2326f3e16cba0084498452dbd599006a6d2a03d2e27d13740f31cc614","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a7b40\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-9p9r4tp77mqp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/158b0ab1-39ac-4c7a-941e-10b6ef2c0d2e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/158b0ab1-39ac-4c7a-941e-10b6ef2c0d2e\",\n \"name\": \"158b0ab1-39ac-4c7a-941e-10b6ef2c0d2e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:59.316278573Z\",\n \"endTime\": \"2026-08-13T13:46:07.406827701Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:98]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a7b40\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-9p9r4tp77mqp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/158b0ab1-39ac-4c7a-941e-10b6ef2c0d2e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/158b0ab1-39ac-4c7a-941e-10b6ef2c0d2e\",\n \"name\": \"158b0ab1-39ac-4c7a-941e-10b6ef2c0d2e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:59.316278573Z\",\n \"endTime\": \"2026-08-13T13:46:07.406827701Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"242a6d8d92d4640315d0a257f95cc4b3b7973a1fed2f5fc0c02eb6e02ca5b45c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"b332eacdf6ea48b33f57793593fd00c2a7f0e83eb6ab4135ca119dcff4706dbc","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-xhfcxz\" with install version 4.20.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0a460\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-xhfcxz: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xhfcxz\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-kvqwbhrf2kpw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1e7d27f7-dd88-416d-85e1-0205607adbdf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1e7d27f7-dd88-416d-85e1-0205607adbdf\",\n \"name\": \"1e7d27f7-dd88-416d-85e1-0205607adbdf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:55.856240586Z\",\n \"endTime\": \"2026-08-13T13:45:57.078463715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-xhfcxz\" with install version 4.20.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0a460\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-xhfcxz: failed waiting for cluster=\"cp-ystream-upgrade-4-21-xhfcxz\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-kvqwbhrf2kpw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1e7d27f7-dd88-416d-85e1-0205607adbdf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1e7d27f7-dd88-416d-85e1-0205607adbdf\",\n \"name\": \"1e7d27f7-dd88-416d-85e1-0205607adbdf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:55.856240586Z\",\n \"endTime\": \"2026-08-13T13:45:57.078463715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"26588e74087427e7aea6535ac57b7e440f2aaecb5e7594811828b818fc602bf5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"dfc17cec067ab4b2e66735b77d4c61f6dc5374c99f12a2e2ae80dd113cf13e37","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b98260\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-zctzkfxv64zx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/90899442-dda9-4298-a7ba-25a7fd4d7edf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/90899442-dda9-4298-a7ba-25a7fd4d7edf\",\n \"name\": \"90899442-dda9-4298-a7ba-25a7fd4d7edf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:09.567795826Z\",\n \"endTime\": \"2026-08-13T12:54:25.005538003Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b98260\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-zctzkfxv64zx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/90899442-dda9-4298-a7ba-25a7fd4d7edf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/90899442-dda9-4298-a7ba-25a7fd4d7edf\",\n \"name\": \"90899442-dda9-4298-a7ba-25a7fd4d7edf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:09.567795826Z\",\n \"endTime\": \"2026-08-13T12:54:25.005538003Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"267841c28ccf8bc3362aab5a2f0bd81b7a6ad476b913c095f85d90bc65272d64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"ee2c2563fe930c718c0fc4b848650ad0233fc9148a41a1d6257d79e9e7c79e30","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001538580\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-kvpzkwp94fv7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fbeca0be-1423-4fa5-ba2f-dd33d188a8cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fbeca0be-1423-4fa5-ba2f-dd33d188a8cd\",\n \"name\": \"fbeca0be-1423-4fa5-ba2f-dd33d188a8cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:34:53.263099757Z\",\n \"endTime\": \"2026-08-13T12:53:55.450561167Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001538580\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-kvpzkwp94fv7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fbeca0be-1423-4fa5-ba2f-dd33d188a8cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fbeca0be-1423-4fa5-ba2f-dd33d188a8cd\",\n \"name\": \"fbeca0be-1423-4fa5-ba2f-dd33d188a8cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:34:53.263099757Z\",\n \"endTime\": \"2026-08-13T12:53:55.450561167Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2ff0a91390bed09ab1bb6c527fb5d9ee1e878d9eba1a98a8375178df2438b57e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"f07f590c5a8560ff8119cd24252b714d16c80267fe27a479f40dd4fb4cf17c16","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004af000\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-4xs9zblwkvtz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0d1b369e-0647-4378-acc6-ef9f1a8f3242\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0d1b369e-0647-4378-acc6-ef9f1a8f3242\",\n \"name\": \"0d1b369e-0647-4378-acc6-ef9f1a8f3242\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:34:59.431542965Z\",\n \"endTime\": \"2026-08-13T12:54:04.906588213Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004af000\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-4xs9zblwkvtz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0d1b369e-0647-4378-acc6-ef9f1a8f3242\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0d1b369e-0647-4378-acc6-ef9f1a8f3242\",\n \"name\": \"0d1b369e-0647-4378-acc6-ef9f1a8f3242\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:34:59.431542965Z\",\n \"endTime\": \"2026-08-13T12:54:04.906588213Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"37e1a7142166db8872fd686f50e647682ad894e8d7db02b5b1822b8756485b12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"425be683392c0c74a3bbcfa76c1c0889b2af39ebb84f9950eb5ae5dd1d976ccd","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cde080\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jct449tngnrp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e342bea8-10f8-4c0f-b501-b5ae5c075cf7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e342bea8-10f8-4c0f-b501-b5ae5c075cf7\",\n \"name\": \"e342bea8-10f8-4c0f-b501-b5ae5c075cf7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:17:59.61601564Z\",\n \"endTime\": \"2026-08-13T14:37:00.733446523Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cde080\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-jct449tngnrp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e342bea8-10f8-4c0f-b501-b5ae5c075cf7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e342bea8-10f8-4c0f-b501-b5ae5c075cf7\",\n \"name\": \"e342bea8-10f8-4c0f-b501-b5ae5c075cf7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:17:59.61601564Z\",\n \"endTime\": \"2026-08-13T14:37:00.733446523Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"443f41caf2141092dcb0b5e589bbdff22fddae2e53f98a1a3bc6e31ccd016614","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"fe3d89e47ac90ac143320e51c687bc46c5041419622db40d299e6bbba749611e","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a5e20\u003e: \n failed to create HCP cluster ea-listx9qc2t: failed waiting for cluster=\"ea-listx9qc2t\" in resourcegroup=\"ea-list-l4mqfz5p9v76\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2381407d-ab80-4deb-b778-c0606c043b06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2381407d-ab80-4deb-b778-c0606c043b06\",\n \"name\": \"2381407d-ab80-4deb-b778-c0606c043b06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:52.639213013Z\",\n \"endTime\": \"2026-08-13T13:46:06.54383697Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a5e20\u003e: \n failed to create HCP cluster ea-listx9qc2t: failed waiting for cluster=\"ea-listx9qc2t\" in resourcegroup=\"ea-list-l4mqfz5p9v76\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2381407d-ab80-4deb-b778-c0606c043b06\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2381407d-ab80-4deb-b778-c0606c043b06\",\n \"name\": \"2381407d-ab80-4deb-b778-c0606c043b06\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:52.639213013Z\",\n \"endTime\": \"2026-08-13T13:46:06.54383697Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4d2df178a7ce6f2d89517184cf9603937d12443432e9a3f81b4794fd5c692e6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"aec8701d23db866ae4cea382008cc5c9a79ecf31566fd5fd40a80d0539d11818","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-bflxfdg7x6rt/cluster-candidate-4-20-xfm5qv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d03860\u003e: \n failed to create HCP cluster cluster-candidate-4-20-xfm5qv: failed waiting for cluster=\"cluster-candidate-4-20-xfm5qv\" in resourcegroup=\"rg-candidate-4-20-bflxfdg7x6rt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0d9f3916-adf5-4210-90e9-5c40e2805301\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0d9f3916-adf5-4210-90e9-5c40e2805301\",\n \"name\": \"0d9f3916-adf5-4210-90e9-5c40e2805301\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:25.712835641Z\",\n \"endTime\": \"2026-08-13T13:45:27.241403611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-bflxfdg7x6rt/cluster-candidate-4-20-xfm5qv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d03860\u003e: \n failed to create HCP cluster cluster-candidate-4-20-xfm5qv: failed waiting for cluster=\"cluster-candidate-4-20-xfm5qv\" in resourcegroup=\"rg-candidate-4-20-bflxfdg7x6rt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0d9f3916-adf5-4210-90e9-5c40e2805301\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0d9f3916-adf5-4210-90e9-5c40e2805301\",\n \"name\": \"0d9f3916-adf5-4210-90e9-5c40e2805301\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:25.712835641Z\",\n \"endTime\": \"2026-08-13T13:45:27.241403611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4fbb44a4b1f655f4c20f1b01c9cd5cde7233eec4d306a7d99e77b283de7bba16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"d335cd7d56763119aaf2ae0bfa63da4b818d848de6700bc4e65e49a7639fd405","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-kph49x with version 4.21.28\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001082500\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-kph49x: failed waiting for cluster=\"np-version-upgrade-cluster-kph49x\" in resourcegroup=\"rg-np-version-upgrade-kph49x-qzv2tc9hdhjs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e78c4f65-41af-4092-a107-8b440e94cc9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e78c4f65-41af-4092-a107-8b440e94cc9b\",\n \"name\": \"e78c4f65-41af-4092-a107-8b440e94cc9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:09:56.342795728Z\",\n \"endTime\": \"2026-08-13T14:29:09.788307741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-kph49x with version 4.21.28\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001082500\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-kph49x: failed waiting for cluster=\"np-version-upgrade-cluster-kph49x\" in resourcegroup=\"rg-np-version-upgrade-kph49x-qzv2tc9hdhjs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e78c4f65-41af-4092-a107-8b440e94cc9b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e78c4f65-41af-4092-a107-8b440e94cc9b\",\n \"name\": \"e78c4f65-41af-4092-a107-8b440e94cc9b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:09:56.342795728Z\",\n \"endTime\": \"2026-08-13T14:29:09.788307741Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5334b58b55c3104dda5961dae00e7e708c9649fc70f8e4187e98feab0e2841f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"a42ddb62e29cf0025c5bf982292f3974b36a5cbe56eeef9956e3eda26efc8d22","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e81240\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-8fqz947nlsp2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0900c2fb-24f3-422d-a87b-749c2dce1d1f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0900c2fb-24f3-422d-a87b-749c2dce1d1f\",\n \"name\": \"0900c2fb-24f3-422d-a87b-749c2dce1d1f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:53.414362961Z\",\n \"endTime\": \"2026-08-13T13:46:06.601863916Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e81240\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-8fqz947nlsp2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0900c2fb-24f3-422d-a87b-749c2dce1d1f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0900c2fb-24f3-422d-a87b-749c2dce1d1f\",\n \"name\": \"0900c2fb-24f3-422d-a87b-749c2dce1d1f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:53.414362961Z\",\n \"endTime\": \"2026-08-13T13:46:06.601863916Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"54906567a1fe5210843f1a155f7c4e0209b006fa8180eb248c05314a07dcd158","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"9923ba6ff4c141da7c0325a9780e2b9ee671cb6a111f5dd3b6a6fae3895c835a","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001205c0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-7zsldfhl96cx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2731387f-3501-4335-aed3-fbf1abf57509\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2731387f-3501-4335-aed3-fbf1abf57509\",\n \"name\": \"2731387f-3501-4335-aed3-fbf1abf57509\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:24.557375305Z\",\n \"endTime\": \"2026-08-13T12:54:24.67925463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001205c0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-7zsldfhl96cx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2731387f-3501-4335-aed3-fbf1abf57509\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2731387f-3501-4335-aed3-fbf1abf57509\",\n \"name\": \"2731387f-3501-4335-aed3-fbf1abf57509\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:24.557375305Z\",\n \"endTime\": \"2026-08-13T12:54:24.67925463Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"557ed7fd4327353665535d46cdc6b077b8dd85e81c37f83a28126a031cca285b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"d98b2626efa4fca035fec43c17b508a41d2c22542d7c5d100f4e7bc76cfe9680","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001786b10\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c554bba3-5631-4cda-9802-6484e1a61569\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c554bba3-5631-4cda-9802-6484e1a61569\",\n \"name\": \"c554bba3-5631-4cda-9802-6484e1a61569\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:36:08.699163008Z\",\n \"endTime\": \"2026-08-13T12:55:24.434741357Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001786b10\u003e: \n GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c554bba3-5631-4cda-9802-6484e1a61569\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c554bba3-5631-4cda-9802-6484e1a61569\",\n \"name\": \"c554bba3-5631-4cda-9802-6484e1a61569\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:36:08.699163008Z\",\n \"endTime\": \"2026-08-13T12:55:24.434741357Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"61aa2d5defa1546a36f9988a2a2bde3e97f01e961377ba5f99da1779fbc858cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"ac777b95864ccc727bf73b81fba7efcbcb834d99c7a83a5d67206444f047b209","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1e760\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-m8ztm8bc5mjx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e60078ec-175f-42c6-abbc-227aa5d61a1c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e60078ec-175f-42c6-abbc-227aa5d61a1c\",\n \"name\": \"e60078ec-175f-42c6-abbc-227aa5d61a1c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:18:34.970587586Z\",\n \"endTime\": \"2026-08-13T14:37:40.370118365Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1e760\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-m8ztm8bc5mjx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e60078ec-175f-42c6-abbc-227aa5d61a1c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e60078ec-175f-42c6-abbc-227aa5d61a1c\",\n \"name\": \"e60078ec-175f-42c6-abbc-227aa5d61a1c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:18:34.970587586Z\",\n \"endTime\": \"2026-08-13T14:37:40.370118365Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"61b223bfcfe5e997bd153cfe11f8d6758eb57f888175db3aaf3196e7de93ecc8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"ae4f2d93567f4d177b92e19ba66d3be7de5e37d61f60b2ac11a00bcc5ae24e63","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-86ppw9\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088fa40\u003e: \n failed to create HCP cluster np-noedge-86ppw9: failed waiting for cluster=\"np-noedge-86ppw9\" in resourcegroup=\"rg-np-noedge-86ppw9-8gd5jkmf69vq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a1342fc6-022d-4f7e-a9b9-891e896ee9f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a1342fc6-022d-4f7e-a9b9-891e896ee9f1\",\n \"name\": \"a1342fc6-022d-4f7e-a9b9-891e896ee9f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:17:59.401077876Z\",\n \"endTime\": \"2026-08-13T14:37:00.72930664Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-86ppw9\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088fa40\u003e: \n failed to create HCP cluster np-noedge-86ppw9: failed waiting for cluster=\"np-noedge-86ppw9\" in resourcegroup=\"rg-np-noedge-86ppw9-8gd5jkmf69vq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a1342fc6-022d-4f7e-a9b9-891e896ee9f1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a1342fc6-022d-4f7e-a9b9-891e896ee9f1\",\n \"name\": \"a1342fc6-022d-4f7e-a9b9-891e896ee9f1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:17:59.401077876Z\",\n \"endTime\": \"2026-08-13T14:37:00.72930664Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"62737560690ddd01a1bee85d3cc4cef5f90527545a449636cacc6686bacec92a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"4e35251d59aa38391eacf4ed5d56476274f302b1101c8009ae40d2840a7b020e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-2c6cr5f4dbr2/cluster-candidate-4-21-p2cv54 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a25180\u003e: \n failed to create HCP cluster cluster-candidate-4-21-p2cv54: failed waiting for cluster=\"cluster-candidate-4-21-p2cv54\" in resourcegroup=\"rg-candidate-4-21-2c6cr5f4dbr2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a1dc60f3-f5d2-4f8e-8a72-0d05e0e6f090\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a1dc60f3-f5d2-4f8e-8a72-0d05e0e6f090\",\n \"name\": \"a1dc60f3-f5d2-4f8e-8a72-0d05e0e6f090\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:23.07296539Z\",\n \"endTime\": \"2026-08-13T13:45:37.855654204Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-21-2c6cr5f4dbr2/cluster-candidate-4-21-p2cv54 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a25180\u003e: \n failed to create HCP cluster cluster-candidate-4-21-p2cv54: failed waiting for cluster=\"cluster-candidate-4-21-p2cv54\" in resourcegroup=\"rg-candidate-4-21-2c6cr5f4dbr2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a1dc60f3-f5d2-4f8e-8a72-0d05e0e6f090\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a1dc60f3-f5d2-4f8e-8a72-0d05e0e6f090\",\n \"name\": \"a1dc60f3-f5d2-4f8e-8a72-0d05e0e6f090\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:23.07296539Z\",\n \"endTime\": \"2026-08-13T13:45:37.855654204Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"645329fe1c238822569c735528c5fbf54ab560810e229b44b0e9b20063e9aa85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"1672b823d5bb91129bcdf629a4bfd1361e098826a6e9cc1f8082529f3f9791ad","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001f86f00\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fst58bwgz7np\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c2157648-4336-474c-bfaf-52608e28607d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c2157648-4336-474c-bfaf-52608e28607d\",\n \"name\": \"c2157648-4336-474c-bfaf-52608e28607d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:57.877053143Z\",\n \"endTime\": \"2026-08-13T13:45:57.039213286Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001f86f00\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-fst58bwgz7np\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c2157648-4336-474c-bfaf-52608e28607d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c2157648-4336-474c-bfaf-52608e28607d\",\n \"name\": \"c2157648-4336-474c-bfaf-52608e28607d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:57.877053143Z\",\n \"endTime\": \"2026-08-13T13:45:57.039213286Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"668643083fe63b0c48353351c2e98e51a65fe0f2bba7f35eaf99de026aa7947d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"712565660509c471aac497011b6194673b46130351ac7c2426ddd7f50f09b079","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a3a640\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-pvq228qjp6k6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0967e016-4d0e-41d8-907b-427fd802a952\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0967e016-4d0e-41d8-907b-427fd802a952\",\n \"name\": \"0967e016-4d0e-41d8-907b-427fd802a952\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:09.697066835Z\",\n \"endTime\": \"2026-08-13T12:54:14.388625352Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:90]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a3a640\u003e: \n failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-pvq228qjp6k6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0967e016-4d0e-41d8-907b-427fd802a952\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/0967e016-4d0e-41d8-907b-427fd802a952\",\n \"name\": \"0967e016-4d0e-41d8-907b-427fd802a952\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:09.697066835Z\",\n \"endTime\": \"2026-08-13T12:54:14.388625352Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"673819f667ef2618c8ac71f0a454fb462cb8b30cc79b1f8819453a451f8be325","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"1b6bb5ed4c2ea791022ddbaee83504568cf7d9c494953007b9739f644614aa10","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-84bfxc at version 4.22.9\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005af6e0\u003e: \n failed to create HCP cluster np-skip-minor-84bfxc: failed waiting for cluster=\"np-skip-minor-84bfxc\" in resourcegroup=\"rg-np-skip-minor-84bfxc-v5vcr2r2wlh6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d88d3ae3-e884-4645-a7b9-3c8d769f6966\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d88d3ae3-e884-4645-a7b9-3c8d769f6966\",\n \"name\": \"d88d3ae3-e884-4645-a7b9-3c8d769f6966\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:17:58.694323926Z\",\n \"endTime\": \"2026-08-13T14:37:01.46212293Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-84bfxc at version 4.22.9\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005af6e0\u003e: \n failed to create HCP cluster np-skip-minor-84bfxc: failed waiting for cluster=\"np-skip-minor-84bfxc\" in resourcegroup=\"rg-np-skip-minor-84bfxc-v5vcr2r2wlh6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d88d3ae3-e884-4645-a7b9-3c8d769f6966\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d88d3ae3-e884-4645-a7b9-3c8d769f6966\",\n \"name\": \"d88d3ae3-e884-4645-a7b9-3c8d769f6966\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:17:58.694323926Z\",\n \"endTime\": \"2026-08-13T14:37:01.46212293Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6e837078706ca8321c073074a5a6ecf5a2921321c34d7c536ea43f2f36671fd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"570ed92628f58ce24785fabbda78656146af9560440d48ec970e8f14e5ee28d4","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:593]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1ee40\u003e: \n failed to create HCP cluster np-downgrade-wldvfz: failed waiting for cluster=\"np-downgrade-wldvfz\" in resourcegroup=\"rg-np-downgrade-wldvfz-hhwkvfq9nbfd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3c3196c3-f55a-4f93-8edd-5700ee138772\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3c3196c3-f55a-4f93-8edd-5700ee138772\",\n \"name\": \"3c3196c3-f55a-4f93-8edd-5700ee138772\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:18:23.65892791Z\",\n \"endTime\": \"2026-08-13T14:37:30.897024914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:593]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1ee40\u003e: \n failed to create HCP cluster np-downgrade-wldvfz: failed waiting for cluster=\"np-downgrade-wldvfz\" in resourcegroup=\"rg-np-downgrade-wldvfz-hhwkvfq9nbfd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3c3196c3-f55a-4f93-8edd-5700ee138772\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3c3196c3-f55a-4f93-8edd-5700ee138772\",\n \"name\": \"3c3196c3-f55a-4f93-8edd-5700ee138772\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:18:23.65892791Z\",\n \"endTime\": \"2026-08-13T14:37:30.897024914Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"80510b2b89c2eaa90d23d3fa2644f8a92412df48c126858c9cfdc73d03a2860f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"baf99862696f1ceabf3420f805862dbaaabef70f1ef0346cf338ca1a715659bb","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dc2a20\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-h65bq5xb7zpb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/7acd20c6-9c30-40ed-86c3-352dc15e741a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/7acd20c6-9c30-40ed-86c3-352dc15e741a\",\n \"name\": \"7acd20c6-9c30-40ed-86c3-352dc15e741a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:27:02.949048295Z\",\n \"endTime\": \"2026-08-13T13:46:07.325052258Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dc2a20\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-h65bq5xb7zpb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/7acd20c6-9c30-40ed-86c3-352dc15e741a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/7acd20c6-9c30-40ed-86c3-352dc15e741a\",\n \"name\": \"7acd20c6-9c30-40ed-86c3-352dc15e741a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:27:02.949048295Z\",\n \"endTime\": \"2026-08-13T13:46:07.325052258Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"875310fa2bc0e7da4d011f2c2dc2ef09492a8b385ac8e58556093e6d255dc1b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"b7a5492787b566bbf536961e0da3c15e740ed4aa66ba50105f398fb23abee884","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-mxfqfc\" with version 4.20.32 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001078720\u003e: \n failed to create HCP cluster cluster-zstream-4-20-mxfqfc: failed waiting for cluster=\"cluster-zstream-4-20-mxfqfc\" in resourcegroup=\"rg-zstream-upgrade-4-20-5jgx5r85nfwn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/827288d5-0031-40e8-a607-b13f26ffb3b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/827288d5-0031-40e8-a607-b13f26ffb3b9\",\n \"name\": \"827288d5-0031-40e8-a607-b13f26ffb3b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:23.639338839Z\",\n \"endTime\": \"2026-08-13T13:45:27.254796453Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-mxfqfc\" with version 4.20.32 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001078720\u003e: \n failed to create HCP cluster cluster-zstream-4-20-mxfqfc: failed waiting for cluster=\"cluster-zstream-4-20-mxfqfc\" in resourcegroup=\"rg-zstream-upgrade-4-20-5jgx5r85nfwn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/827288d5-0031-40e8-a607-b13f26ffb3b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/827288d5-0031-40e8-a607-b13f26ffb3b9\",\n \"name\": \"827288d5-0031-40e8-a607-b13f26ffb3b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:23.639338839Z\",\n \"endTime\": \"2026-08-13T13:45:27.254796453Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8a8cfd96ff7a9020ffc8fd9d4f1847b04d905c1a81fa35fe1938d07c28cc3e84","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"a0944cbb3759f21fd5aca543eb03916918011ad34b27ffa59dc9a6ce1fec0a02","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001204980\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-h8d4xpdzbt9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1cf189cd-f4b7-41f2-acb7-634e2cede823\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1cf189cd-f4b7-41f2-acb7-634e2cede823\",\n \"name\": \"1cf189cd-f4b7-41f2-acb7-634e2cede823\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:36:20.275674618Z\",\n \"endTime\": \"2026-08-13T12:55:25.206207969Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001204980\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-h8d4xpdzbt9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1cf189cd-f4b7-41f2-acb7-634e2cede823\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1cf189cd-f4b7-41f2-acb7-634e2cede823\",\n \"name\": \"1cf189cd-f4b7-41f2-acb7-634e2cede823\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:36:20.275674618Z\",\n \"endTime\": \"2026-08-13T12:55:25.206207969Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8eb9b468a2a5d4074a813beca4ad9d06d2da4a3ca353ece5ad443a2ca523a580","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"4ffa0e21d93f5b8744bf5d71747ced5f30b82b6965f0f26a528a4f3fc0cdcf44","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0ee00\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7r9zhl78pj92\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/961e76ae-0417-4ead-a246-62b3b25dc88a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/961e76ae-0417-4ead-a246-62b3b25dc88a\",\n \"name\": \"961e76ae-0417-4ead-a246-62b3b25dc88a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:34:59.668231839Z\",\n \"endTime\": \"2026-08-13T12:54:05.498497391Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0ee00\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-7r9zhl78pj92\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/961e76ae-0417-4ead-a246-62b3b25dc88a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/961e76ae-0417-4ead-a246-62b3b25dc88a\",\n \"name\": \"961e76ae-0417-4ead-a246-62b3b25dc88a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:34:59.668231839Z\",\n \"endTime\": \"2026-08-13T12:54:05.498497391Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"93512c82a2d46b6c9cb36815febfce826abcc43bf955764cf108d0ee7e75f5ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"d712d5ce17264f0ef550496665e2a7bd7d67dfac479ff00deed5bcb5e610f7c9","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"9651c0066ca069bed890b43d1ff7c223c8522db753223162d3d95992f2d933c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"5e9c061b227b9a0c906c7efbd9762f8060022c94e57582895b25b5bfb6464c8e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001452500\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-4c78kjdlcvrf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2d41f4bf-8edf-43bb-9286-92a9fbc1a1a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2d41f4bf-8edf-43bb-9286-92a9fbc1a1a0\",\n \"name\": \"2d41f4bf-8edf-43bb-9286-92a9fbc1a1a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:27:04.542928314Z\",\n \"endTime\": \"2026-08-13T13:46:06.612609369Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001452500\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-4c78kjdlcvrf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2d41f4bf-8edf-43bb-9286-92a9fbc1a1a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/2d41f4bf-8edf-43bb-9286-92a9fbc1a1a0\",\n \"name\": \"2d41f4bf-8edf-43bb-9286-92a9fbc1a1a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:27:04.542928314Z\",\n \"endTime\": \"2026-08-13T13:46:06.612609369Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"99256c05f133acfdad3582b5b2cefb5f214bdbdb8a8983a6582b4a5feb36500c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"a6003df04dcefb46b248b60ceda04a32a6dcb374cd38b2b36a2dfd3d0e9d3a92","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db6a20\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-wwqdrlxljwwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/b5988061-6dae-4ae1-bb0a-bb5f69208556\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/b5988061-6dae-4ae1-bb0a-bb5f69208556\",\n \"name\": \"b5988061-6dae-4ae1-bb0a-bb5f69208556\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:01.428456562Z\",\n \"endTime\": \"2026-08-13T12:54:14.639225182Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db6a20\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-wwqdrlxljwwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/b5988061-6dae-4ae1-bb0a-bb5f69208556\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/b5988061-6dae-4ae1-bb0a-bb5f69208556\",\n \"name\": \"b5988061-6dae-4ae1-bb0a-bb5f69208556\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:01.428456562Z\",\n \"endTime\": \"2026-08-13T12:54:14.639225182Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9a1f587e7722dbe7b9e42ff46fbf3d4eb605a7f5bb1a8ebecee75e524cca51b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"ed2445a883ec43a5f79308aae46436db53967da9b3d7e4549128c5dc72d3a5ea","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-19-7nb496\" with version 4.19.40 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa0f60\u003e: \n failed to create HCP cluster cluster-zstream-4-19-7nb496: failed waiting for cluster=\"cluster-zstream-4-19-7nb496\" in resourcegroup=\"rg-zstream-upgrade-4-19-xfv562tx8dm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5f214755-f002-43aa-b3d1-e2f63643cfaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5f214755-f002-43aa-b3d1-e2f63643cfaf\",\n \"name\": \"5f214755-f002-43aa-b3d1-e2f63643cfaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:33.372177413Z\",\n \"endTime\": \"2026-08-13T13:45:38.108196402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-19-7nb496\" with version 4.19.40 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa0f60\u003e: \n failed to create HCP cluster cluster-zstream-4-19-7nb496: failed waiting for cluster=\"cluster-zstream-4-19-7nb496\" in resourcegroup=\"rg-zstream-upgrade-4-19-xfv562tx8dm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5f214755-f002-43aa-b3d1-e2f63643cfaf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5f214755-f002-43aa-b3d1-e2f63643cfaf\",\n \"name\": \"5f214755-f002-43aa-b3d1-e2f63643cfaf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:33.372177413Z\",\n \"endTime\": \"2026-08-13T13:45:38.108196402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a5a9e56484e4727d3583b51db61bc182972bac9771e6d081dd46e009ab8a7a4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"203b46a994d5d317fdb59bdd7dc81eba37cbecacc4f1ddd570442a328e2be045","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-v8d459 with version 4.21.28\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e93980\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-v8d459: failed waiting for cluster=\"np-version-upgrade-cluster-v8d459\" in resourcegroup=\"rg-np-version-upgrade-v8d459-l2xnj768t5vj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/9a8c6782-43a4-4fed-8a17-2e579ab6473f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/9a8c6782-43a4-4fed-8a17-2e579ab6473f\",\n \"name\": \"9a8c6782-43a4-4fed-8a17-2e579ab6473f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:47:28.173153409Z\",\n \"endTime\": \"2026-08-13T14:06:28.769423438Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-v8d459 with version 4.21.28\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e93980\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-v8d459: failed waiting for cluster=\"np-version-upgrade-cluster-v8d459\" in resourcegroup=\"rg-np-version-upgrade-v8d459-l2xnj768t5vj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/9a8c6782-43a4-4fed-8a17-2e579ab6473f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/9a8c6782-43a4-4fed-8a17-2e579ab6473f\",\n \"name\": \"9a8c6782-43a4-4fed-8a17-2e579ab6473f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:47:28.173153409Z\",\n \"endTime\": \"2026-08-13T14:06:28.769423438Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a6d9fc807ecc8777be9118ce683367f7ea104c4af83bbc95e52b36b686452391","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"4983ace40cc218681b7857e0423a3923e3cd26d87c1638d400339c6f41432348","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f00360\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wr9d7c2p6w7d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1445df4e-c34d-4c61-bf18-854f57eebf1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1445df4e-c34d-4c61-bf18-854f57eebf1e\",\n \"name\": \"1445df4e-c34d-4c61-bf18-854f57eebf1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:18:29.278419728Z\",\n \"endTime\": \"2026-08-13T14:37:30.963374449Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f00360\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-wr9d7c2p6w7d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1445df4e-c34d-4c61-bf18-854f57eebf1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1445df4e-c34d-4c61-bf18-854f57eebf1e\",\n \"name\": \"1445df4e-c34d-4c61-bf18-854f57eebf1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:18:29.278419728Z\",\n \"endTime\": \"2026-08-13T14:37:30.963374449Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ad827ee5906e011173d72515ec5ff73add82d9f1451c990a7be6f5701ce2b193","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"01a31292cd515dc8f519670b243d3abfd4119a9483f99307a8bd96fae20c7525","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:741]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d92a80\u003e: \n failed to create HCP cluster np-dg-minor-pghz8v: failed waiting for cluster=\"np-dg-minor-pghz8v\" in resourcegroup=\"rg-np-dg-minor-pghz8v-lrw5sgmh8jl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\",\n \"name\": \"1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:17:58.32975278Z\",\n \"endTime\": \"2026-08-13T14:37:00.365479085Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:741]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d92a80\u003e: \n failed to create HCP cluster np-dg-minor-pghz8v: failed waiting for cluster=\"np-dg-minor-pghz8v\" in resourcegroup=\"rg-np-dg-minor-pghz8v-lrw5sgmh8jl2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\",\n \"name\": \"1fe2ebf3-2ba2-4f3a-9716-d209bd154df6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:17:58.32975278Z\",\n \"endTime\": \"2026-08-13T14:37:00.365479085Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b38c56400a8351661662d1ee9114a109afa7a26cf80f838218ec32af9bc95c03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"9cc879374e15052498fe107f632241d60262d11e582f3cd75ba610c34972a164","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5af60\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-wn4trssh2rw8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/73da94d9-1633-4f7b-9c57-a07e3dcbac5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/73da94d9-1633-4f7b-9c57-a07e3dcbac5e\",\n \"name\": \"73da94d9-1633-4f7b-9c57-a07e3dcbac5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:08.88865423Z\",\n \"endTime\": \"2026-08-13T12:54:14.847054095Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a5af60\u003e: \n failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-wn4trssh2rw8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/73da94d9-1633-4f7b-9c57-a07e3dcbac5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/73da94d9-1633-4f7b-9c57-a07e3dcbac5e\",\n \"name\": \"73da94d9-1633-4f7b-9c57-a07e3dcbac5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:08.88865423Z\",\n \"endTime\": \"2026-08-13T12:54:14.847054095Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b41a38bad694c1aa0aa8e7747cf3db6627d537bd5b11a27961505778b731651e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"58a7665130289bc8bf8b38124a9d7256cf3067a08ec04ee7abba242447a0f9ad","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-hlxg86hwqwzd/cluster-candidate-4-22-bzdgws should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ab260\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bzdgws: failed waiting for cluster=\"cluster-candidate-4-22-bzdgws\" in resourcegroup=\"rg-candidate-4-22-hlxg86hwqwzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c7a01a0a-086b-44e8-b1a5-31adc8390dfc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c7a01a0a-086b-44e8-b1a5-31adc8390dfc\",\n \"name\": \"c7a01a0a-086b-44e8-b1a5-31adc8390dfc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:22.62882439Z\",\n \"endTime\": \"2026-08-13T13:45:27.580496533Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-22-hlxg86hwqwzd/cluster-candidate-4-22-bzdgws should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ab260\u003e: \n failed to create HCP cluster cluster-candidate-4-22-bzdgws: failed waiting for cluster=\"cluster-candidate-4-22-bzdgws\" in resourcegroup=\"rg-candidate-4-22-hlxg86hwqwzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c7a01a0a-086b-44e8-b1a5-31adc8390dfc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c7a01a0a-086b-44e8-b1a5-31adc8390dfc\",\n \"name\": \"c7a01a0a-086b-44e8-b1a5-31adc8390dfc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:22.62882439Z\",\n \"endTime\": \"2026-08-13T13:45:27.580496533Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b566af3e8c64eff6e924d7f3bea057023ca473ad0126b880aac3b066292cba93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"75c093f2bd0988ddc22af9e2f1728a3622a04d8d837f81fc566043f24916c7e1","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:151]: failed to create HCP cluster version 4.19\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0016d02a0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-zksnvzwlwp47\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-zksnvzwlwp47, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:151]: failed to create HCP cluster version 4.19\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0016d02a0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-zksnvzwlwp47\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-zksnvzwlwp47, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-jxrdzr, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: failed waiting for cluster=\"cluster-nightly-5-1-jxrdzr\" in resourcegroup=\"rg-nightly-5-1-ps74xj8djqv9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jxrdzr in resource group rg-nightly-5-1-ps74xj8djqv9, error: context deadline exceeded"}},{"row_id":"ba238cfad8a976cb20b0c404dba2566cbad3faa9ec5baa3a24861846d15e3648","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"b306455abc970335937504a0b8dbf970342a6cddaf65ead1d47c1cd26ac8e503","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-gcx662\" with version 4.22.6 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0a700\u003e: \n failed to create HCP cluster cluster-zstream-4-22-gcx662: failed waiting for cluster=\"cluster-zstream-4-22-gcx662\" in resourcegroup=\"rg-zstream-upgrade-4-22-xhpjgfhqnmk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1f5418ad-11cf-4854-9c55-f5009d5093e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1f5418ad-11cf-4854-9c55-f5009d5093e5\",\n \"name\": \"1f5418ad-11cf-4854-9c55-f5009d5093e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:50.747428938Z\",\n \"endTime\": \"2026-08-13T13:45:56.980878141Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-gcx662\" with version 4.22.6 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0a700\u003e: \n failed to create HCP cluster cluster-zstream-4-22-gcx662: failed waiting for cluster=\"cluster-zstream-4-22-gcx662\" in resourcegroup=\"rg-zstream-upgrade-4-22-xhpjgfhqnmk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1f5418ad-11cf-4854-9c55-f5009d5093e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1f5418ad-11cf-4854-9c55-f5009d5093e5\",\n \"name\": \"1f5418ad-11cf-4854-9c55-f5009d5093e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:50.747428938Z\",\n \"endTime\": \"2026-08-13T13:45:56.980878141Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c6698971fb0ea44ebf9a754408394eb302ef86cb4636cf1915bcb6e75183e903","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"fb68c0506afca9dbb7157bf21fe830abf97281976b1bd452bf568279ca022ce2","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-n2z8tx with version 4.20.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009006e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-n2z8tx: failed waiting for cluster=\"np-version-upgrade-cluster-n2z8tx\" in resourcegroup=\"rg-np-version-upgrade-n2z8tx-h6rxn4wxdf2d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/9864b614-3413-435e-bb89-4680ab1d12d6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/9864b614-3413-435e-bb89-4680ab1d12d6\",\n \"name\": \"9864b614-3413-435e-bb89-4680ab1d12d6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:15:21.131304407Z\",\n \"endTime\": \"2026-08-13T14:34:21.117716397Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-n2z8tx with version 4.20.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009006e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-n2z8tx: failed waiting for cluster=\"np-version-upgrade-cluster-n2z8tx\" in resourcegroup=\"rg-np-version-upgrade-n2z8tx-h6rxn4wxdf2d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/9864b614-3413-435e-bb89-4680ab1d12d6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/9864b614-3413-435e-bb89-4680ab1d12d6\",\n \"name\": \"9864b614-3413-435e-bb89-4680ab1d12d6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T14:15:21.131304407Z\",\n \"endTime\": \"2026-08-13T14:34:21.117716397Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cf5d2591251637b258d0fabbc139115093fe678142785c9a4d0afa1157d1688f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"e98c722279619b6f44c7448b6d51b5e98d3de275d4306b57bec8a68c891b8553","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000901400\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-r4jmjnt5xljx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/ac6daceb-a51f-40b4-a6ea-2af296d27b7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/ac6daceb-a51f-40b4-a6ea-2af296d27b7b\",\n \"name\": \"ac6daceb-a51f-40b4-a6ea-2af296d27b7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:57.662691592Z\",\n \"endTime\": \"2026-08-13T13:45:57.037404294Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000901400\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-r4jmjnt5xljx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/ac6daceb-a51f-40b4-a6ea-2af296d27b7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/ac6daceb-a51f-40b4-a6ea-2af296d27b7b\",\n \"name\": \"ac6daceb-a51f-40b4-a6ea-2af296d27b7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:57.662691592Z\",\n \"endTime\": \"2026-08-13T13:45:57.037404294Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d70d5cef91445a8be6218eb619c425fd2085e5550300f80661ed5c60f3ba54d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"4ae69f49ae8aa5592b66ab89cbf16ca51a19caab61da63ea271f0cd5d3df03a5","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008709e0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hx9r9jt8dhs9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/8e92c75a-8a94-4d8c-9c3f-45c4131fd2ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/8e92c75a-8a94-4d8c-9c3f-45c4131fd2ae\",\n \"name\": \"8e92c75a-8a94-4d8c-9c3f-45c4131fd2ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:07.389897181Z\",\n \"endTime\": \"2026-08-13T12:54:14.650522734Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008709e0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hx9r9jt8dhs9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/8e92c75a-8a94-4d8c-9c3f-45c4131fd2ae\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/8e92c75a-8a94-4d8c-9c3f-45c4131fd2ae\",\n \"name\": \"8e92c75a-8a94-4d8c-9c3f-45c4131fd2ae\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:07.389897181Z\",\n \"endTime\": \"2026-08-13T12:54:14.650522734Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"db58191fe79af474b07d430bbffc4f12b5b0967c61871d198e28da801c910017","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"45888832c8fee65bfcc0a82bc8bb932e5906d8f25722fc7e89cc811f9eb0813b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-vlsdjsrqk7lw/cluster-candidate-5-0-rsqgdw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d64d00\u003e: \n failed to create HCP cluster cluster-candidate-5-0-rsqgdw: failed waiting for cluster=\"cluster-candidate-5-0-rsqgdw\" in resourcegroup=\"rg-candidate-5-0-vlsdjsrqk7lw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fa30f8ce-7d6e-47b0-ae53-31bfb6cd84c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fa30f8ce-7d6e-47b0-ae53-31bfb6cd84c1\",\n \"name\": \"fa30f8ce-7d6e-47b0-ae53-31bfb6cd84c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:35.997920598Z\",\n \"endTime\": \"2026-08-13T13:45:37.891483648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-vlsdjsrqk7lw/cluster-candidate-5-0-rsqgdw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d64d00\u003e: \n failed to create HCP cluster cluster-candidate-5-0-rsqgdw: failed waiting for cluster=\"cluster-candidate-5-0-rsqgdw\" in resourcegroup=\"rg-candidate-5-0-vlsdjsrqk7lw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fa30f8ce-7d6e-47b0-ae53-31bfb6cd84c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fa30f8ce-7d6e-47b0-ae53-31bfb6cd84c1\",\n \"name\": \"fa30f8ce-7d6e-47b0-ae53-31bfb6cd84c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:26:35.997920598Z\",\n \"endTime\": \"2026-08-13T13:45:37.891483648Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e5725642d4067ffb5f44aec7761360a001267557ad2547672863aefca22fdce0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"4de42c0a3f9058e9fc17d4498714fc862d677cabc34be3a326710e7585ed6749","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db2de0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2j5l4h4wwg48\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/70ed4f6b-6d80-4dce-aafb-faca01cbd7d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/70ed4f6b-6d80-4dce-aafb-faca01cbd7d7\",\n \"name\": \"70ed4f6b-6d80-4dce-aafb-faca01cbd7d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:36:41.080128266Z\",\n \"endTime\": \"2026-08-13T12:55:45.4952121Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db2de0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2j5l4h4wwg48\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/70ed4f6b-6d80-4dce-aafb-faca01cbd7d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/70ed4f6b-6d80-4dce-aafb-faca01cbd7d7\",\n \"name\": \"70ed4f6b-6d80-4dce-aafb-faca01cbd7d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:36:41.080128266Z\",\n \"endTime\": \"2026-08-13T12:55:45.4952121Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ee40e20810454a0074ee4f090f3761ccf7a89fca8bfd4c6d2176aa9dc5b240ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"c7fb8276f580095e6f8ef823c7dde1335bc3adadbbd1dc163b3f43085da14b7e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0f1e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jt6gl2lqksvm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/b54c0b33-bbf1-4bd8-a667-684f39fb3c0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/b54c0b33-bbf1-4bd8-a667-684f39fb3c0c\",\n \"name\": \"b54c0b33-bbf1-4bd8-a667-684f39fb3c0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:36:44.671472486Z\",\n \"endTime\": \"2026-08-13T12:55:44.418018361Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0f1e0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jt6gl2lqksvm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/b54c0b33-bbf1-4bd8-a667-684f39fb3c0c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/b54c0b33-bbf1-4bd8-a667-684f39fb3c0c\",\n \"name\": \"b54c0b33-bbf1-4bd8-a667-684f39fb3c0c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:36:44.671472486Z\",\n \"endTime\": \"2026-08-13T12:55:44.418018361Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f08ba8e0882d3894c16d82980b0aa241749ae18ee16c1321f4b27cfc2e1ed9f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"a4add8c8ab59e6141afc7b6e22addd6d022fc04df75730792951cdb459e0490c","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9f160\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-pmb29qtdz5lq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1366a2a3-5ab5-4f06-bfe6-0f5becb13d1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1366a2a3-5ab5-4f06-bfe6-0f5becb13d1e\",\n \"name\": \"1366a2a3-5ab5-4f06-bfe6-0f5becb13d1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:04.456571151Z\",\n \"endTime\": \"2026-08-13T12:54:14.663583178Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9f160\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-pmb29qtdz5lq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1366a2a3-5ab5-4f06-bfe6-0f5becb13d1e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1366a2a3-5ab5-4f06-bfe6-0f5becb13d1e\",\n \"name\": \"1366a2a3-5ab5-4f06-bfe6-0f5becb13d1e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:04.456571151Z\",\n \"endTime\": \"2026-08-13T12:54:14.663583178Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f3f40a4cf179bc4d2f385e43e9b6c80d681f91291a7f8982a209434b72f7220d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"710f2c56702d12963db949df115d707494485a0cd1fc368fc7b342aa1d4243c2","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004af160\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-pnvgx29s75xl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/25607779-31f0-48c6-bc0a-70d1e5af2fad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/25607779-31f0-48c6-bc0a-70d1e5af2fad\",\n \"name\": \"25607779-31f0-48c6-bc0a-70d1e5af2fad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:01.197413261Z\",\n \"endTime\": \"2026-08-13T12:54:14.437329044Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004af160\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-pnvgx29s75xl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/25607779-31f0-48c6-bc0a-70d1e5af2fad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/25607779-31f0-48c6-bc0a-70d1e5af2fad\",\n \"name\": \"25607779-31f0-48c6-bc0a-70d1e5af2fad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:35:01.197413261Z\",\n \"endTime\": \"2026-08-13T12:54:14.437329044Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fb2f91014d8ccd609ce9aaa8897c2cbb36ef17c3e141e3d59a82b8e8b10212a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087873183595106304","occurred_at":"2026-08-13T12:05:32Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed"}}],"failure_pattern_summary":{"signature_count":47,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b"],"matched_occurrences":47,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-13T11:53:01Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"31b55bc5219fc761d859357191d13f500e37fa554cfc869c309337d148b4af53","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","failed_at":"e2e","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"633d834b26f6a7ead43c5efac04b6ae8e0122889088ae291c487974f5850fac0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","failed_at":"e2e","signature_id":"71d707d8d230945319eacc89a89423b1ed0d8831a6f5ba1ad76723820292b2d8","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:197]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00131aa00\u003e: \n failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-xtxbpdvbqtqq\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:197]: failed to create node pool version 4.19.7 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00131aa00\u003e: \n failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-xtxbpdvbqtqq\" to finish creating: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-xtxbpdvbqtqq\" to finish creating: context deadline exceeded"}},{"row_id":"733e8e22dbbd6fd5f80cc2823863167cb4f0c52c40435343db27109750a03eef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","failed_at":"e2e","signature_id":"0421de590218865da65b3af252c9dfa3cd7c3f55271342d96e5089c485c514a1","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d209e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-5mmdptfkb4qp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/024da1c7-6df0-487c-8a47-bb2fc6fb63cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/024da1c7-6df0-487c-8a47-bb2fc6fb63cc\",\n \"name\": \"024da1c7-6df0-487c-8a47-bb2fc6fb63cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T11:57:40.70442981Z\",\n \"endTime\": \"2026-08-13T12:16:46.395053793Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d209e0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-5mmdptfkb4qp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/024da1c7-6df0-487c-8a47-bb2fc6fb63cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/024da1c7-6df0-487c-8a47-bb2fc6fb63cc\",\n \"name\": \"024da1c7-6df0-487c-8a47-bb2fc6fb63cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T11:57:40.70442981Z\",\n \"endTime\": \"2026-08-13T12:16:46.395053793Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79d42a294edfd7eaa19516eb5e39cb2fbbc57c83345983aea3a63ac716c4f9fa","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"897dd0075437db651163def535606a5ec40779212a8125152de32d396bd48c42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","failed_at":"e2e","signature_id":"de9fba52a960b5d509c226c1c083c6d0b28085e513bcaecc67e116c6984c9df4","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f60480\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-x78c7dcw45gl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f775432f-7cac-483a-af3d-cdc0bc7c5113\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f775432f-7cac-483a-af3d-cdc0bc7c5113\",\n \"name\": \"f775432f-7cac-483a-af3d-cdc0bc7c5113\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:57:19.306229714Z\",\n \"endTime\": \"2026-08-13T13:16:27.918607796Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f60480\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-x78c7dcw45gl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f775432f-7cac-483a-af3d-cdc0bc7c5113\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/f775432f-7cac-483a-af3d-cdc0bc7c5113\",\n \"name\": \"f775432f-7cac-483a-af3d-cdc0bc7c5113\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:57:19.306229714Z\",\n \"endTime\": \"2026-08-13T13:16:27.918607796Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d10399ad24d5a13d7845e9b64657f41c22a702880e4a4d9cb1142f7520a6868","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","failed_at":"e2e","signature_id":"5b50bdacf46db4a62ac301c86303aa6cdc04641a80f7a89024a59e1b84faa061","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e84520\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-drrx6cd6559n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b56ccafd-4383-4ad9-9f24-2989a66e180f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b56ccafd-4383-4ad9-9f24-2989a66e180f\",\n \"name\": \"b56ccafd-4383-4ad9-9f24-2989a66e180f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T11:56:40.932047182Z\",\n \"endTime\": \"2026-08-13T12:15:46.19337299Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:103]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e84520\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-drrx6cd6559n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b56ccafd-4383-4ad9-9f24-2989a66e180f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b56ccafd-4383-4ad9-9f24-2989a66e180f\",\n \"name\": \"b56ccafd-4383-4ad9-9f24-2989a66e180f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T11:56:40.932047182Z\",\n \"endTime\": \"2026-08-13T12:15:46.19337299Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"90bf4a72abb67fd7cc7f868499a01fbd84a611dee73fd99a1f5636e5ac783d20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","failed_at":"e2e","signature_id":"438feed7d6aa6c7b27f44db5135ec2496eb739806679ce21b0a68d1334a6abc8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-cpr92p6jj5qj/cluster-candidate-4-20-vsc8hd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00086a380\u003e: \n failed to create HCP cluster cluster-candidate-4-20-vsc8hd: failed waiting for cluster=\"cluster-candidate-4-20-vsc8hd\" in resourcegroup=\"rg-candidate-4-20-cpr92p6jj5qj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/106422c2-56db-4b2a-99d7-c603a17c5e6a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/106422c2-56db-4b2a-99d7-c603a17c5e6a\",\n \"name\": \"106422c2-56db-4b2a-99d7-c603a17c5e6a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:27:57.532451241Z\",\n \"endTime\": \"2026-08-13T12:46:57.159906172Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-cpr92p6jj5qj/cluster-candidate-4-20-vsc8hd should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00086a380\u003e: \n failed to create HCP cluster cluster-candidate-4-20-vsc8hd: failed waiting for cluster=\"cluster-candidate-4-20-vsc8hd\" in resourcegroup=\"rg-candidate-4-20-cpr92p6jj5qj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/106422c2-56db-4b2a-99d7-c603a17c5e6a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/106422c2-56db-4b2a-99d7-c603a17c5e6a\",\n \"name\": \"106422c2-56db-4b2a-99d7-c603a17c5e6a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:27:57.532451241Z\",\n \"endTime\": \"2026-08-13T12:46:57.159906172Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"95edf8a3ca4213696d402d0c307e30b50fc6188c6f02615a91a9c1cb58f49742","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","failed_at":"e2e","signature_id":"32533af6ce35e8298dd5671118e3af98dc4c24311c09f33c4046c07eb485d35b","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc0027b4a98\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fzfll.apps.aro.d9s7k9e5c7c4b4u.01ef.westeurope.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc0027b4a98\u003e: \n VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-fzfll.apps.aro.d9s7k9e5c7c4b4u.01ef.westeurope.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"aba867d880670203c92175c02ae8d6e19e0f59d244b9b84a1a8000e6a093760e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","failed_at":"e2e","signature_id":"4ea026269c6c2776cf2d008e8e33582e8c4d04ad5a222a3c4eda1afa2cacb4af","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-mm6p45mrvjqv/cluster-candidate-5-0-7ql8bc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e01820\u003e: \n failed to create HCP cluster cluster-candidate-5-0-7ql8bc: failed waiting for cluster=\"cluster-candidate-5-0-7ql8bc\" in resourcegroup=\"rg-candidate-5-0-mm6p45mrvjqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/34962d53-53be-48ee-b96e-e2fa47155f69\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/34962d53-53be-48ee-b96e-e2fa47155f69\",\n \"name\": \"34962d53-53be-48ee-b96e-e2fa47155f69\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:30:12.239477716Z\",\n \"endTime\": \"2026-08-13T12:49:17.224447305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-5-0-mm6p45mrvjqv/cluster-candidate-5-0-7ql8bc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e01820\u003e: \n failed to create HCP cluster cluster-candidate-5-0-7ql8bc: failed waiting for cluster=\"cluster-candidate-5-0-7ql8bc\" in resourcegroup=\"rg-candidate-5-0-mm6p45mrvjqv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/34962d53-53be-48ee-b96e-e2fa47155f69\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/34962d53-53be-48ee-b96e-e2fa47155f69\",\n \"name\": \"34962d53-53be-48ee-b96e-e2fa47155f69\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:30:12.239477716Z\",\n \"endTime\": \"2026-08-13T12:49:17.224447305Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb8ca43a79863985fe5fa47ef1bbe80d7bc25200d089461928007bb1c0bcd82f","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d131feaa354d426b3545c394ec877c1e527b0abe32a2c9c9a03787ebd560bd58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","failed_at":"e2e","signature_id":"2fc25500d7bff95fdcd99958c0b17bb722c42af6f861b04da0d5491067520d71","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:759]: failed to create nodepool npdg-421-419\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011f8a20\u003e: \n failed to create NodePool npdg-421-419: failed waiting for nodepool=\"npdg-421-419\" for cluster \"np-dg-minor-pr5df2\" in resourcegroup=\"rg-np-dg-minor-pr5df2-4xk5xp2xhfm2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/879ca928-cb37-4db6-81e7-46a882044a50\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/879ca928-cb37-4db6-81e7-46a882044a50\",\n \"name\": \"879ca928-cb37-4db6-81e7-46a882044a50\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:16:32.109830139Z\",\n \"endTime\": \"2026-08-13T13:35:38.839757933Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:759]: failed to create nodepool npdg-421-419\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011f8a20\u003e: \n failed to create NodePool npdg-421-419: failed waiting for nodepool=\"npdg-421-419\" for cluster \"np-dg-minor-pr5df2\" in resourcegroup=\"rg-np-dg-minor-pr5df2-4xk5xp2xhfm2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/879ca928-cb37-4db6-81e7-46a882044a50\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/879ca928-cb37-4db6-81e7-46a882044a50\",\n \"name\": \"879ca928-cb37-4db6-81e7-46a882044a50\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T13:16:32.109830139Z\",\n \"endTime\": \"2026-08-13T13:35:38.839757933Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fc770accebb7d6c912b410b81bd939c4dd51d1283a4d23a3af99a3db2766cc75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087870036076138496","occurred_at":"2026-08-13T11:53:01Z","failed_at":"e2e","signature_id":"53ea9738f993f6a47e2afe9b16549b79000eb355a3b59cb715117ac5adabcd88","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e87200\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-26s6gwvg7mfr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c0acbe1d-54f4-4da8-bfb9-dd26516663bc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c0acbe1d-54f4-4da8-bfb9-dd26516663bc\",\n \"name\": \"c0acbe1d-54f4-4da8-bfb9-dd26516663bc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:41:06.178659568Z\",\n \"endTime\": \"2026-08-13T13:00:07.891456178Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e87200\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-26s6gwvg7mfr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c0acbe1d-54f4-4da8-bfb9-dd26516663bc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/c0acbe1d-54f4-4da8-bfb9-dd26516663bc\",\n \"name\": \"c0acbe1d-54f4-4da8-bfb9-dd26516663bc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T12:41:06.178659568Z\",\n \"endTime\": \"2026-08-13T13:00:07.891456178Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","67e9146bba6a263169b2663e0ece2eff489af41d3e74f4e915f7ce1b74288068","79d42a294edfd7eaa19516eb5e39cb2fbbc57c83345983aea3a63ac716c4f9fa","79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","cb8ca43a79863985fe5fa47ef1bbe80d7bc25200d089461928007bb1c0bcd82f"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087827320206266368","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-13T09:03:17Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"777cef57519f028b8b3545465ac9990c34f9173e6816ea3c6d89020c118a2204","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087827320206266368","occurred_at":"2026-08-13T09:03:17Z","failed_at":"e2e","signature_id":"d712d5ce17264f0ef550496665e2a7bd7d67dfac479ff00deed5bcb5e610f7c9","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"cee74024792bc46a8296f19b853d6380ca34dad67de84d709bf7e37b51cadabb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087827320206266368","occurred_at":"2026-08-13T09:03:17Z","failed_at":"e2e","signature_id":"7896e80f3d43b9ff9e34ed42eb43701faf6b382200f9cda7c5ea503730f39acb","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:152]: failed to create node pool npupgrade-4-20 with version 4.20.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e91280\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-4f46mm\" in resourcegroup=\"rg-np-version-upgrade-4f46mm-wzwtwbsl9hk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/29565f62-084a-40db-9076-e7ac51f80f6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/29565f62-084a-40db-9076-e7ac51f80f6d\",\n \"name\": \"29565f62-084a-40db-9076-e7ac51f80f6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T10:53:13.351872344Z\",\n \"endTime\": \"2026-08-13T11:12:13.375278226Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:152]: failed to create node pool npupgrade-4-20 with version 4.20.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e91280\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-4f46mm\" in resourcegroup=\"rg-np-version-upgrade-4f46mm-wzwtwbsl9hk4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/29565f62-084a-40db-9076-e7ac51f80f6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/29565f62-084a-40db-9076-e7ac51f80f6d\",\n \"name\": \"29565f62-084a-40db-9076-e7ac51f80f6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T10:53:13.351872344Z\",\n \"endTime\": \"2026-08-13T11:12:13.375278226Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d4f45410636b967ea633738e866e3484811348ae423940c7757cef8773235c13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087827320206266368","occurred_at":"2026-08-13T09:03:17Z","failed_at":"e2e","signature_id":"8bd705a5eb53385882f688bf9ed38b343f1f635530b9a0d3639f8756cd03121b","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001515260\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-q7xxx5pzfppj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3d3a25d8-6e45-49e1-9192-bac778144e7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3d3a25d8-6e45-49e1-9192-bac778144e7d\",\n \"name\": \"3d3a25d8-6e45-49e1-9192-bac778144e7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T10:33:53.189451766Z\",\n \"endTime\": \"2026-08-13T10:53:02.074442054Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001515260\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-q7xxx5pzfppj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3d3a25d8-6e45-49e1-9192-bac778144e7d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3d3a25d8-6e45-49e1-9192-bac778144e7d\",\n \"name\": \"3d3a25d8-6e45-49e1-9192-bac778144e7d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-13T10:33:53.189451766Z\",\n \"endTime\": \"2026-08-13T10:53:02.074442054Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087826220786913280","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-13T08:58:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2087775454810345472","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-13T05:37:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2087720951423700992","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-13T02:00:37Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}