{"meta":{"date":"2026-08-06","timezone":"UTC","generated_at":"2026-08-26T07:39:17Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":20,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":17,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085501203734597632","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-06T23:00:07Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2e96dd73c36c3529883a846f945b8ef552f65808c63155ec4cf19d23bb8c02fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085501203734597632","occurred_at":"2026-08-06T23:00:07Z","failed_at":"e2e","signature_id":"9a5a1829f3ce94f2f253b46e318acac897ea906fab88e6b2b85a5ffdb6d75ecd","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002780840\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-272j4k46kvrl, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-272j4k46kvrl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-272j4k46kvrl, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002780840\u003e: \n failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-272j4k46kvrl, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-272j4k46kvrl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-272j4k46kvrl, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-272j4k46kvrl, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-272j4k46kvrl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-272j4k46kvrl, error: context deadline exceeded"}},{"row_id":"58390208ffeb3692304e62af802dc1f406ed34ee12203225a93e294efb7a4309","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085501203734597632","occurred_at":"2026-08-06T23:00:07Z","failed_at":"e2e","signature_id":"7ba46c2f23dd67bf50b9c6f00c7043d6b6dbdd71455c7c119db41361c923025b","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a30180\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-nkkjqglbtkz6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f40d5702-8f9c-44c7-adc1-5d00590f3971\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: NoRegisteredProviderFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"NoRegisteredProviderFound\",\n \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:120]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a30180\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-nkkjqglbtkz6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f40d5702-8f9c-44c7-adc1-5d00590f3971\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: NoRegisteredProviderFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"NoRegisteredProviderFound\",\n \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24660fe0387a8714cdbcf9260ba9d42cd88ae6f8a2d64bc65cfcc351b75b07c4","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'locations/hcpOperationStatuses'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound"}},{"row_id":"a57aacbaa58e8b05d5d4ccff93c3adb5172c6e3bf1f7a27b2bab4444aa4d15eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085501203734597632","occurred_at":"2026-08-06T23:00:07Z","failed_at":"e2e","signature_id":"4769652dcdd434db15b0f9f2f568efb4eb42cbef122eae222dda1c6c411aa2b5","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ad7e0\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-z7fl6rs7cjg5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b557842c-42a5-4d65-80d1-401050420a8d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b557842c-42a5-4d65-80d1-401050420a8d\",\n \"name\": \"b557842c-42a5-4d65-80d1-401050420a8d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T23:42:49.092257041Z\",\n \"endTime\": \"2026-08-07T00:02:00.144502022Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009ad7e0\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-z7fl6rs7cjg5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b557842c-42a5-4d65-80d1-401050420a8d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b557842c-42a5-4d65-80d1-401050420a8d\",\n \"name\": \"b557842c-42a5-4d65-80d1-401050420a8d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T23:42:49.092257041Z\",\n \"endTime\": \"2026-08-07T00:02:00.144502022Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f62ba8e5444e9c223a230eff7f1c4f0648bec06eb85ad653cd7bbb616c636e02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2085501203734597632","occurred_at":"2026-08-06T23:00:07Z","failed_at":"e2e","signature_id":"2e5d7357142344a3caecc974d360925ce1edc5c37e5458306f994b1ef066b558","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc00171c228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-wvptdr\" in resourcegroup=\"rg-zstream-upgrade-4-21-bvlh6ndvpmbz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8813fe71-a996-402a-8b22-5a2a07f190dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8813fe71-a996-402a-8b22-5a2a07f190dc\",\n \"name\": \"8813fe71-a996-402a-8b22-5a2a07f190dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T23:41:52.408330345Z\",\n \"endTime\": \"2026-08-07T00:06:00.233442021Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s1cmcbdh4et6l3hlfi6mpmgpaik517c still exists (deletion dispatched at 2026-08-06T23:41:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc00171c228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-wvptdr\" in resourcegroup=\"rg-zstream-upgrade-4-21-bvlh6ndvpmbz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8813fe71-a996-402a-8b22-5a2a07f190dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8813fe71-a996-402a-8b22-5a2a07f190dc\",\n \"name\": \"8813fe71-a996-402a-8b22-5a2a07f190dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T23:41:52.408330345Z\",\n \"endTime\": \"2026-08-07T00:06:00.233442021Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s1cmcbdh4et6l3hlfi6mpmgpaik517c still exists (deletion dispatched at 2026-08-06T23:41:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6d43e4ab2fe017bbc54d307366c534735e562004e9108078bd8d55ba38e56fb","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","24660fe0387a8714cdbcf9260ba9d42cd88ae6f8a2d64bc65cfcc351b75b07c4","36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","f6d43e4ab2fe017bbc54d307366c534735e562004e9108078bd8d55ba38e56fb"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085480339634917376","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T21:37:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085440461937315840","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T18:58:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085440284472119296","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T18:58:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085439389160181760","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-06T18:54:30Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a03c6bd5a741fb02a7bcdaad6436753cdcb0c2584c3eff7b7f6b1961d51ebe3c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085439389160181760","occurred_at":"2026-08-06T18:54:30Z","failed_at":"e2e","signature_id":"16a035304592e71ce5eed76cf1a021ca48547d113592b6be98c79407f46e9e74","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000e90288\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-2tgsqq2jndsw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/60668361-99db-4918-b1e5-1dd336777928\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/60668361-99db-4918-b1e5-1dd336777928\",\n \"name\": \"60668361-99db-4918-b1e5-1dd336777928\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T19:32:55.875583035Z\",\n \"endTime\": \"2026-08-06T19:51:56.785806181Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:328]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000e90288\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-2tgsqq2jndsw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/60668361-99db-4918-b1e5-1dd336777928\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/60668361-99db-4918-b1e5-1dd336777928\",\n \"name\": \"60668361-99db-4918-b1e5-1dd336777928\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T19:32:55.875583035Z\",\n \"endTime\": \"2026-08-06T19:51:56.785806181Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085389830853758976","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T14:08:38Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085377974613315584","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T14:04:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085366360346464256","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T14:02:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085365042831691776","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T13:56:02Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-06T12:27:57Z"},"failed_at":["e2e"],"failed_test_count":29,"occurrences":[{"row_id":"085c67c4fcfc12faaf8415151c766430290c00b1260a82c993ffa530369f6087","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"d7d4632730204a64bbe64c7ed12cc3f61a340b62b2997f48c9c30126f44f6b2a","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e21180\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-wm5j8g7bxx2m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/47987963-3e83-4032-867d-1e91993059bf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/47987963-3e83-4032-867d-1e91993059bf\",\n \"name\": \"47987963-3e83-4032-867d-1e91993059bf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:06:15.739222219Z\",\n \"endTime\": \"2026-08-06T14:25:32.992085923Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:217]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e21180\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-wm5j8g7bxx2m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/47987963-3e83-4032-867d-1e91993059bf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/47987963-3e83-4032-867d-1e91993059bf\",\n \"name\": \"47987963-3e83-4032-867d-1e91993059bf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:06:15.739222219Z\",\n \"endTime\": \"2026-08-06T14:25:32.992085923Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1b6416f2f004457fa19a8819e3aee91a5d2af85d9e08c98be82c675ff2d9f92c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"55cdb2e9385dc3b91f8cbacff5d9608844b3aa8edfe35103ec1619200bb37a21","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-h8m8gn at version 4.22.8\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ba160\u003e: \n failed to create HCP cluster np-skip-minor-h8m8gn: failed waiting for cluster=\"np-skip-minor-h8m8gn\" in resourcegroup=\"rg-np-skip-minor-h8m8gn-mmthppt2pm7w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/045886da-6fff-4fbb-938f-87cdb4e85bad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/045886da-6fff-4fbb-938f-87cdb4e85bad\",\n \"name\": \"045886da-6fff-4fbb-938f-87cdb4e85bad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:02:42.413063766Z\",\n \"endTime\": \"2026-08-06T14:21:42.765300575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:462]: failed to create HCP cluster np-skip-minor-h8m8gn at version 4.22.8\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ba160\u003e: \n failed to create HCP cluster np-skip-minor-h8m8gn: failed waiting for cluster=\"np-skip-minor-h8m8gn\" in resourcegroup=\"rg-np-skip-minor-h8m8gn-mmthppt2pm7w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/045886da-6fff-4fbb-938f-87cdb4e85bad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/045886da-6fff-4fbb-938f-87cdb4e85bad\",\n \"name\": \"045886da-6fff-4fbb-938f-87cdb4e85bad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:02:42.413063766Z\",\n \"endTime\": \"2026-08-06T14:21:42.765300575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2664ba698e065133d63570eafe36e9eb11f4b40082e5be1290a958c24947c755","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"236fba08ee6513ef627e3344c7d63f6b4e48e780c89c1407f0125e055897a74d","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de74e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-ts8pl2pswsxq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/17591ace-40e3-4096-ab61-8932d25ea783\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/17591ace-40e3-4096-ab61-8932d25ea783\",\n \"name\": \"17591ace-40e3-4096-ab61-8932d25ea783\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:27:45.606306967Z\",\n \"endTime\": \"2026-08-06T13:46:50.497502059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de74e0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-ts8pl2pswsxq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/17591ace-40e3-4096-ab61-8932d25ea783\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/17591ace-40e3-4096-ab61-8932d25ea783\",\n \"name\": \"17591ace-40e3-4096-ab61-8932d25ea783\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:27:45.606306967Z\",\n \"endTime\": \"2026-08-06T13:46:50.497502059Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"287d6ab9fd37fa2788bbc680c24dea9120d8391dc0d3a6be848d1a7329220a6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"ec7b912aaaba2ddbc3ce4c059ee6de5f7386b7169e4e245236d674d3fe8eafa3","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:741]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000686c20\u003e: \n failed to create HCP cluster np-dg-minor-rlg5t7: failed waiting for cluster=\"np-dg-minor-rlg5t7\" in resourcegroup=\"rg-np-dg-minor-rlg5t7-rrc9jm2hdf6d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fdf3be45-0611-45f9-93a3-33c90642bf28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fdf3be45-0611-45f9-93a3-33c90642bf28\",\n \"name\": \"fdf3be45-0611-45f9-93a3-33c90642bf28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:03:30.092901342Z\",\n \"endTime\": \"2026-08-06T14:22:32.789219091Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:741]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000686c20\u003e: \n failed to create HCP cluster np-dg-minor-rlg5t7: failed waiting for cluster=\"np-dg-minor-rlg5t7\" in resourcegroup=\"rg-np-dg-minor-rlg5t7-rrc9jm2hdf6d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fdf3be45-0611-45f9-93a3-33c90642bf28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/fdf3be45-0611-45f9-93a3-33c90642bf28\",\n \"name\": \"fdf3be45-0611-45f9-93a3-33c90642bf28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:03:30.092901342Z\",\n \"endTime\": \"2026-08-06T14:22:32.789219091Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2afec5d89952e9a80defed0c33898ef0272a1cfe209f7bfb0957a9ea8cbf1a28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"b5a8313535e2abca1b3751a59805bec039aa97ee9b5c1005f43c575033f63ac4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-tg2v4gkq6vnr/cluster-candidate-4-22-p7kqt2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f10c40\u003e: \n failed to create HCP cluster cluster-candidate-4-22-p7kqt2: failed waiting for cluster=\"cluster-candidate-4-22-p7kqt2\" in resourcegroup=\"rg-candidate-4-22-tg2v4gkq6vnr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb16ace1-2bdd-451a-9d78-7250d76ae24b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb16ace1-2bdd-451a-9d78-7250d76ae24b\",\n \"name\": \"eb16ace1-2bdd-451a-9d78-7250d76ae24b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:14:36.88824918Z\",\n \"endTime\": \"2026-08-06T13:33:40.344451949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:121]: HCP cluster rg-candidate-4-22-tg2v4gkq6vnr/cluster-candidate-4-22-p7kqt2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f10c40\u003e: \n failed to create HCP cluster cluster-candidate-4-22-p7kqt2: failed waiting for cluster=\"cluster-candidate-4-22-p7kqt2\" in resourcegroup=\"rg-candidate-4-22-tg2v4gkq6vnr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb16ace1-2bdd-451a-9d78-7250d76ae24b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb16ace1-2bdd-451a-9d78-7250d76ae24b\",\n \"name\": \"eb16ace1-2bdd-451a-9d78-7250d76ae24b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:14:36.88824918Z\",\n \"endTime\": \"2026-08-06T13:33:40.344451949Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [hypershiftHostedCluster] hosted cluster has no installed version; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"326bbb50ac6108b5d1e7c67911892ff1f9352e5ea4d7d5aed66ca3727b2c2bd4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"9d516d7683d88b693761cce380780674bca380e614ab83ae644074e4f8409476","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-v22lmz with version 4.20.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880740\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-v22lmz: failed waiting for cluster=\"np-version-upgrade-cluster-v22lmz\" in resourcegroup=\"rg-np-version-upgrade-v22lmz-dv89nwqtszzk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b382466b-61e3-429b-a11e-9e3842470fc6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b382466b-61e3-429b-a11e-9e3842470fc6\",\n \"name\": \"b382466b-61e3-429b-a11e-9e3842470fc6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:50:27.651550221Z\",\n \"endTime\": \"2026-08-06T14:09:31.872279557Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-v22lmz with version 4.20.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880740\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-v22lmz: failed waiting for cluster=\"np-version-upgrade-cluster-v22lmz\" in resourcegroup=\"rg-np-version-upgrade-v22lmz-dv89nwqtszzk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b382466b-61e3-429b-a11e-9e3842470fc6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b382466b-61e3-429b-a11e-9e3842470fc6\",\n \"name\": \"b382466b-61e3-429b-a11e-9e3842470fc6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:50:27.651550221Z\",\n \"endTime\": \"2026-08-06T14:09:31.872279557Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"477c52a479ee1c374ed0376873e15de96ca94cf6fb9c1818a5be586467fc748c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"4c707ac016204c8e21bf0ad6090e04ec80173a388db068cdbe57c97cc89eb577","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9b6a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-ccwxckwntr6l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/68677096-9fc1-47ab-aa3f-959f9dd830ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/68677096-9fc1-47ab-aa3f-959f9dd830ad\",\n \"name\": \"68677096-9fc1-47ab-aa3f-959f9dd830ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:24:48.671191592Z\",\n \"endTime\": \"2026-08-06T13:43:50.420986579Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9b6a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-ccwxckwntr6l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/68677096-9fc1-47ab-aa3f-959f9dd830ad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/68677096-9fc1-47ab-aa3f-959f9dd830ad\",\n \"name\": \"68677096-9fc1-47ab-aa3f-959f9dd830ad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:24:48.671191592Z\",\n \"endTime\": \"2026-08-06T13:43:50.420986579Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"51645afeb6aee3fac3c49c7e289fd90403a23d59e100008cf68500777be16468","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"73b260c57d297290e7538f8366ba615328b5c703f508a1fe98a0b4ba6f52f797","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-l2gkzv\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00078b540\u003e: \n failed to create HCP cluster np-noedge-l2gkzv: failed waiting for cluster=\"np-noedge-l2gkzv\" in resourcegroup=\"rg-np-noedge-l2gkzv-5wbj828vxrrx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/01bbf857-2cca-4bc8-8a4e-9a9270d4dc75\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/01bbf857-2cca-4bc8-8a4e-9a9270d4dc75\",\n \"name\": \"01bbf857-2cca-4bc8-8a4e-9a9270d4dc75\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:01:01.527104846Z\",\n \"endTime\": \"2026-08-06T14:20:12.805550311Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:314]: failed to create HCP cluster np-noedge-l2gkzv\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00078b540\u003e: \n failed to create HCP cluster np-noedge-l2gkzv: failed waiting for cluster=\"np-noedge-l2gkzv\" in resourcegroup=\"rg-np-noedge-l2gkzv-5wbj828vxrrx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/01bbf857-2cca-4bc8-8a4e-9a9270d4dc75\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/01bbf857-2cca-4bc8-8a4e-9a9270d4dc75\",\n \"name\": \"01bbf857-2cca-4bc8-8a4e-9a9270d4dc75\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:01:01.527104846Z\",\n \"endTime\": \"2026-08-06T14:20:12.805550311Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"60e4e610f05e9dc98296d903192debe6710233cf3ff9eff6191489236959e607","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"dcc8d5fae49cd5abff9fecaec39fd8acf4d9b6b3cb487e5a440eb9cfb8074f9e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:120]: failed to get admin REST config for cluster \"cluster-zstream-4-19-crm6vw\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000939710\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-crm6vw\" in resourcegroup=\"rg-zstream-upgrade-4-19-2jfbcdvqrzbm\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-crm6vw in resource group rg-zstream-upgrade-4-19-2jfbcdvqrzbm, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:120]: failed to get admin REST config for cluster \"cluster-zstream-4-19-crm6vw\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000939710\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-crm6vw\" in resourcegroup=\"rg-zstream-upgrade-4-19-2jfbcdvqrzbm\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-crm6vw in resource group rg-zstream-upgrade-4-19-2jfbcdvqrzbm, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-candidate-4-21-4tlk2m\" in resourcegroup=\"rg-candidate-4-21-gwbm5ljx4jpp\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-21-4tlk2m in resource group rg-candidate-4-21-gwbm5ljx4jpp, error: context deadline exceeded"}},{"row_id":"621f1c309a598aa94449e10701ff9ffea21101ac56af2ec8f2c051d1b6389011","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"b6c6f188d72774218055eb6315f871d6aeebe7f1db640df0b66aec6dfcd558a6","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001116360\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-jplb8l9bzh7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7303a152-653d-4091-80cf-f62f456fb7b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7303a152-653d-4091-80cf-f62f456fb7b9\",\n \"name\": \"7303a152-653d-4091-80cf-f62f456fb7b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:25:47.63498122Z\",\n \"endTime\": \"2026-08-06T13:44:50.425486913Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001116360\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-jplb8l9bzh7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7303a152-653d-4091-80cf-f62f456fb7b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7303a152-653d-4091-80cf-f62f456fb7b9\",\n \"name\": \"7303a152-653d-4091-80cf-f62f456fb7b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:25:47.63498122Z\",\n \"endTime\": \"2026-08-06T13:44:50.425486913Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6e4841f8fe3854ccd61995cea6845b99bf6c837f968c3c052b2822ee69bcac02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"3a725768636b5db7a428db163677a754ba1bc010b17a1d104cb433e1bf1f102b","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000849d40\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-f2kqg69vsqnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/38be9e5f-4202-47f2-ba3d-563b8721207a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/38be9e5f-4202-47f2-ba3d-563b8721207a\",\n \"name\": \"38be9e5f-4202-47f2-ba3d-563b8721207a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:23:25.670668106Z\",\n \"endTime\": \"2026-08-06T13:42:30.41053575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000849d40\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-f2kqg69vsqnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/38be9e5f-4202-47f2-ba3d-563b8721207a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/38be9e5f-4202-47f2-ba3d-563b8721207a\",\n \"name\": \"38be9e5f-4202-47f2-ba3d-563b8721207a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:23:25.670668106Z\",\n \"endTime\": \"2026-08-06T13:42:30.41053575Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"75b40f006bce4267728273a5640e9cda6315e71530032bac5cdcaf190b75000f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"66db5f10f756daa0e52be78ac19d5c43ea8da173d9c8fd7bfd56943c2c79d417","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:174]: failed to create node pool \"np-zjfs89\" for cluster \"cluster-candidate-4-20-zjfs89\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d491a0\u003e: \n failed to create NodePool np-zjfs89: failed waiting for nodepool=\"np-zjfs89\" for cluster \"cluster-candidate-4-20-zjfs89\" in resourcegroup=\"rg-candidate-4-20-lwxfdj6b82tz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b242787d-f97b-4793-82e7-f4a360d7884c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b242787d-f97b-4793-82e7-f4a360d7884c\",\n \"name\": \"b242787d-f97b-4793-82e7-f4a360d7884c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:22:17.428651995Z\",\n \"endTime\": \"2026-08-06T13:41:30.639404623Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:174]: failed to create node pool \"np-zjfs89\" for cluster \"cluster-candidate-4-20-zjfs89\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d491a0\u003e: \n failed to create NodePool np-zjfs89: failed waiting for nodepool=\"np-zjfs89\" for cluster \"cluster-candidate-4-20-zjfs89\" in resourcegroup=\"rg-candidate-4-20-lwxfdj6b82tz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b242787d-f97b-4793-82e7-f4a360d7884c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b242787d-f97b-4793-82e7-f4a360d7884c\",\n \"name\": \"b242787d-f97b-4793-82e7-f4a360d7884c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:22:17.428651995Z\",\n \"endTime\": \"2026-08-06T13:41:30.639404623Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8bb50734529263843de9e7a94ae885cfd102fda6a3e7d5be4fce073c359b423c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"765870913940abe2b981e937e3349b9a1e460508668804ddd9824f770fc9f076","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001d84300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7zlftg2lbn8z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\",\n \"name\": \"662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:21:44.42927409Z\",\n \"endTime\": \"2026-08-06T13:45:50.758225457Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s139v1qlkvi8boiqf5sq004bbeikr8v still exists (deletion dispatched at 2026-08-06T13:21:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001d84300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7zlftg2lbn8z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\",\n \"name\": \"662e7bf9-43c4-4c69-b7d4-c84c5e38b0ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:21:44.42927409Z\",\n \"endTime\": \"2026-08-06T13:45:50.758225457Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s139v1qlkvi8boiqf5sq004bbeikr8v still exists (deletion dispatched at 2026-08-06T13:21:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dcbb98b30046e6cc7914c7e62bede6790189f033e9f8bb7bf919eebfa6754c56","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: externalAuths, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"926c01e1ee6065b5c3e728497c902d0adf3a5e0d712806c7b00742c8cd4d18a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"5c03fe2235c8d06fb08c278dd75d56842305bea98635eb6b0cfc187099f924be","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9c6a0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-jkhl9hhgd9nc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8eff199a-e62d-435d-aee5-867005cccb86\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8eff199a-e62d-435d-aee5-867005cccb86\",\n \"name\": \"8eff199a-e62d-435d-aee5-867005cccb86\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:26:19.442548042Z\",\n \"endTime\": \"2026-08-06T13:45:30.447966088Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9c6a0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-jkhl9hhgd9nc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8eff199a-e62d-435d-aee5-867005cccb86\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8eff199a-e62d-435d-aee5-867005cccb86\",\n \"name\": \"8eff199a-e62d-435d-aee5-867005cccb86\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:26:19.442548042Z\",\n \"endTime\": \"2026-08-06T13:45:30.447966088Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"939f19f8399a0ff2162ccdfc2ccde235f8891bab254f9a66f49db05114e7967a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"b3f0bdaaeab67fb751adda6e6d84a48a238cd3ce11351f6595273141a8b88bbd","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-zfn6l2\" with version 4.20.28 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d51220\u003e: \n failed to create HCP cluster cluster-zstream-4-20-zfn6l2: failed waiting for cluster=\"cluster-zstream-4-20-zfn6l2\" in resourcegroup=\"rg-zstream-upgrade-4-20-2pwfwd7cj72b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88e04dc1-f997-4ecd-9971-75dc18ced266\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88e04dc1-f997-4ecd-9971-75dc18ced266\",\n \"name\": \"88e04dc1-f997-4ecd-9971-75dc18ced266\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:18:03.306664145Z\",\n \"endTime\": \"2026-08-06T13:37:10.59559601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-20-zfn6l2\" with version 4.20.28 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d51220\u003e: \n failed to create HCP cluster cluster-zstream-4-20-zfn6l2: failed waiting for cluster=\"cluster-zstream-4-20-zfn6l2\" in resourcegroup=\"rg-zstream-upgrade-4-20-2pwfwd7cj72b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88e04dc1-f997-4ecd-9971-75dc18ced266\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/88e04dc1-f997-4ecd-9971-75dc18ced266\",\n \"name\": \"88e04dc1-f997-4ecd-9971-75dc18ced266\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:18:03.306664145Z\",\n \"endTime\": \"2026-08-06T13:37:10.59559601Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"96111bd51240662f8599f95f94cf888cf006630ab9d3fd12ba77aba8780df3f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"43161b5c7fe5a04604aae7c86aad18d5fbd6ded8d83b993c6cb0eddb5c32ae32","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-9dwj48 with version 4.21.27\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fadf20\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-9dwj48: failed waiting for cluster=\"np-version-upgrade-cluster-9dwj48\" in resourcegroup=\"rg-np-version-upgrade-9dwj48-gvwgsnfm4p5r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/77d206f8-ee5c-4108-b10e-e10bed6f02a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/77d206f8-ee5c-4108-b10e-e10bed6f02a1\",\n \"name\": \"77d206f8-ee5c-4108-b10e-e10bed6f02a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:45:47.06733678Z\",\n \"endTime\": \"2026-08-06T14:04:51.795324203Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-9dwj48 with version 4.21.27\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fadf20\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-9dwj48: failed waiting for cluster=\"np-version-upgrade-cluster-9dwj48\" in resourcegroup=\"rg-np-version-upgrade-9dwj48-gvwgsnfm4p5r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/77d206f8-ee5c-4108-b10e-e10bed6f02a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/77d206f8-ee5c-4108-b10e-e10bed6f02a1\",\n \"name\": \"77d206f8-ee5c-4108-b10e-e10bed6f02a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:45:47.06733678Z\",\n \"endTime\": \"2026-08-06T14:04:51.795324203Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"96dc0dc49202dffaa9f2c9f40fa0bd3a86dd5f4ca345e21205cb1be5b7b360d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"5f977a303d985401d848e41a33b7dd9001fd63426df210f486aaa4f0f64723c7","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a1c7e0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-p6k5qdqwc88z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b09a508d-d159-4b59-bd6c-31a88a545881\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b09a508d-d159-4b59-bd6c-31a88a545881\",\n \"name\": \"b09a508d-d159-4b59-bd6c-31a88a545881\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:05:44.41212661Z\",\n \"endTime\": \"2026-08-06T14:24:52.800922454Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:270]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a1c7e0\u003e: \n failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-p6k5qdqwc88z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b09a508d-d159-4b59-bd6c-31a88a545881\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b09a508d-d159-4b59-bd6c-31a88a545881\",\n \"name\": \"b09a508d-d159-4b59-bd6c-31a88a545881\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:05:44.41212661Z\",\n \"endTime\": \"2026-08-06T14:24:52.800922454Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"96fafcd64d033192f454d60aad0ee8d476614731b9a23e7b2d6d799f45d6b0bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"7245e5e986e4b456f99270b5e77b34f9c65e335493e3317ecedca65c0dcf9619","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000b2aa68\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc000b2aa68\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tkgzckfhvn9s\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tkgzckfhvn9s, error: context deadline exceeded"}},{"row_id":"9deae99b6636f92f5a2137b548acaf55a915c8afe87992e117a7e996a4dfb3e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"01c12e6c0d16de52443dd1ee73fd05c2e77edc7a6584fa351fe8a03ccbc410db","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:131]: failed to get admin REST config for cluster \"cluster-candidate-4-21-4tlk2m\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0008ebb00\u003e: \n failed waiting for hcpCluster=\"cluster-candidate-4-21-4tlk2m\" in resourcegroup=\"rg-candidate-4-21-gwbm5ljx4jpp\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-21-4tlk2m in resource group rg-candidate-4-21-gwbm5ljx4jpp, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:131]: failed to get admin REST config for cluster \"cluster-candidate-4-21-4tlk2m\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0008ebb00\u003e: \n failed waiting for hcpCluster=\"cluster-candidate-4-21-4tlk2m\" in resourcegroup=\"rg-candidate-4-21-gwbm5ljx4jpp\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-21-4tlk2m in resource group rg-candidate-4-21-gwbm5ljx4jpp, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"cluster-candidate-4-21-4tlk2m\" in resourcegroup=\"rg-candidate-4-21-gwbm5ljx4jpp\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-21-4tlk2m in resource group rg-candidate-4-21-gwbm5ljx4jpp, error: context deadline exceeded"}},{"row_id":"a557f972fdeb567f7da60594976ea05d86304f36d9ff71a8ae49e002207c8eb8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"77cfad70a10a8d790241b041e10651a57bd31952e4da039a060f83714fb72f7a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-qgwpct\" with version 4.22.6 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a9580\u003e: \n failed to create HCP cluster cluster-zstream-4-22-qgwpct: failed waiting for cluster=\"cluster-zstream-4-22-qgwpct\" in resourcegroup=\"rg-zstream-upgrade-4-22-sq8l54wqsbkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bbc9d45-d885-4a5e-a947-9f6cc88c4261\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bbc9d45-d885-4a5e-a947-9f6cc88c4261\",\n \"name\": \"4bbc9d45-d885-4a5e-a947-9f6cc88c4261\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:19:49.664906309Z\",\n \"endTime\": \"2026-08-06T13:38:50.306742743Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-22-qgwpct\" with version 4.22.6 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001a9580\u003e: \n failed to create HCP cluster cluster-zstream-4-22-qgwpct: failed waiting for cluster=\"cluster-zstream-4-22-qgwpct\" in resourcegroup=\"rg-zstream-upgrade-4-22-sq8l54wqsbkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bbc9d45-d885-4a5e-a947-9f6cc88c4261\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4bbc9d45-d885-4a5e-a947-9f6cc88c4261\",\n \"name\": \"4bbc9d45-d885-4a5e-a947-9f6cc88c4261\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:19:49.664906309Z\",\n \"endTime\": \"2026-08-06T13:38:50.306742743Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b582dac51c4f1cc9a8a006e0fe1cc7894d01f0d3be9851c05b32679504b3d70e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"3596543c1f0a076fc250f32a4b79935d6f90a8021f9e78f19a42f1a51a6b044d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-zpdmdp\" with install version 4.20.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00083a720\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-zpdmdp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-zpdmdp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nt59ffl77spl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5bb285a7-7ce1-4027-9552-2a08322f2e58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5bb285a7-7ce1-4027-9552-2a08322f2e58\",\n \"name\": \"5bb285a7-7ce1-4027-9552-2a08322f2e58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:20:54.106332052Z\",\n \"endTime\": \"2026-08-06T13:40:00.506003922Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:182]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-zpdmdp\" with install version 4.20.32\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00083a720\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-zpdmdp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-zpdmdp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-nt59ffl77spl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5bb285a7-7ce1-4027-9552-2a08322f2e58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5bb285a7-7ce1-4027-9552-2a08322f2e58\",\n \"name\": \"5bb285a7-7ce1-4027-9552-2a08322f2e58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:20:54.106332052Z\",\n \"endTime\": \"2026-08-06T13:40:00.506003922Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c256670a15c44574ba20751ec76eb0c54ff2771bcdced58b30fb1cfa6af23776","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"0b73f221e723983b7bb466871edd22f26516aca06559029bb0093f2257fe4ed6","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00165cba0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-b2xd2cfggh75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1b4f6d3-9523-41a1-b21f-c572ab6386df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1b4f6d3-9523-41a1-b21f-c572ab6386df\",\n \"name\": \"d1b4f6d3-9523-41a1-b21f-c572ab6386df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:06:00.450328897Z\",\n \"endTime\": \"2026-08-06T14:25:02.805147479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00165cba0\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-b2xd2cfggh75\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1b4f6d3-9523-41a1-b21f-c572ab6386df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d1b4f6d3-9523-41a1-b21f-c572ab6386df\",\n \"name\": \"d1b4f6d3-9523-41a1-b21f-c572ab6386df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:06:00.450328897Z\",\n \"endTime\": \"2026-08-06T14:25:02.805147479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e96bd40306e5e9349e71cc790fc075a342d28558629918b6e3f63dfabb65bb8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"13acca32cc25889f23290d3862df2cd6cdd07bf1837b4014dbe05ff5e8eb9223","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-wjpjjj with version 4.21.27\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d24460\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-wjpjjj: failed waiting for cluster=\"np-version-upgrade-cluster-wjpjjj\" in resourcegroup=\"rg-np-version-upgrade-wjpjjj-pc5nbpkjrzdz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/28d75c45-25ff-4278-a246-ebcc78ca1ef0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/28d75c45-25ff-4278-a246-ebcc78ca1ef0\",\n \"name\": \"28d75c45-25ff-4278-a246-ebcc78ca1ef0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:47:14.337423761Z\",\n \"endTime\": \"2026-08-06T14:06:32.076160168Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:133]: failed to create HCP cluster np-version-upgrade-cluster-wjpjjj with version 4.21.27\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d24460\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-wjpjjj: failed waiting for cluster=\"np-version-upgrade-cluster-wjpjjj\" in resourcegroup=\"rg-np-version-upgrade-wjpjjj-pc5nbpkjrzdz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/28d75c45-25ff-4278-a246-ebcc78ca1ef0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/28d75c45-25ff-4278-a246-ebcc78ca1ef0\",\n \"name\": \"28d75c45-25ff-4278-a246-ebcc78ca1ef0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:47:14.337423761Z\",\n \"endTime\": \"2026-08-06T14:06:32.076160168Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ea0f7311e827d588654d18f79f107fcff69bb744cc0a5f51831687d04dd30f9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"6c295ab12b0ab36152d04a1401762b995e3b3de4d73fa6a738669845b30bbab9","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:593]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00086d0c0\u003e: \n failed to create HCP cluster np-downgrade-6ff98p: failed waiting for cluster=\"np-downgrade-6ff98p\" in resourcegroup=\"rg-np-downgrade-6ff98p-2wm7xthgsqdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d67f7fc-7980-4d5a-a602-efc7b0cae3f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d67f7fc-7980-4d5a-a602-efc7b0cae3f8\",\n \"name\": \"5d67f7fc-7980-4d5a-a602-efc7b0cae3f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:01:51.048715965Z\",\n \"endTime\": \"2026-08-06T14:20:52.703469338Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:593]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00086d0c0\u003e: \n failed to create HCP cluster np-downgrade-6ff98p: failed waiting for cluster=\"np-downgrade-6ff98p\" in resourcegroup=\"rg-np-downgrade-6ff98p-2wm7xthgsqdg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d67f7fc-7980-4d5a-a602-efc7b0cae3f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d67f7fc-7980-4d5a-a602-efc7b0cae3f8\",\n \"name\": \"5d67f7fc-7980-4d5a-a602-efc7b0cae3f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T14:01:51.048715965Z\",\n \"endTime\": \"2026-08-06T14:20:52.703469338Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ec5f5377ecf3e3ee44f5dcd63d536a928089c4ff50146b21927fab5388c1b680","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"e2041a19de87ed544e987a8fc38139feb20ea47d9cf0d09871eb8eaead18deb4","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ea6c0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-pqql6hwj4pb6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d53f51b2-3af2-44b5-a959-3b9e5b951e8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d53f51b2-3af2-44b5-a959-3b9e5b951e8f\",\n \"name\": \"d53f51b2-3af2-44b5-a959-3b9e5b951e8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:27:27.054348153Z\",\n \"endTime\": \"2026-08-06T13:46:30.524065583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ea6c0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-pqql6hwj4pb6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d53f51b2-3af2-44b5-a959-3b9e5b951e8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d53f51b2-3af2-44b5-a959-3b9e5b951e8f\",\n \"name\": \"d53f51b2-3af2-44b5-a959-3b9e5b951e8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:27:27.054348153Z\",\n \"endTime\": \"2026-08-06T13:46:30.524065583Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f49a2a6bca111d9abde19b41e8ab6036a2169369950d54d63d2b4d2aaae70215","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"063098a80cbfe3cc4638e71f1686605b92235136cfb5e8b62870202b46bdec68","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880f00\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-pnf2fh5wjwql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8925daa4-519a-42f1-a42f-eda89e216058\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8925daa4-519a-42f1-a42f-eda89e216058\",\n \"name\": \"8925daa4-519a-42f1-a42f-eda89e216058\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:29:35.264600406Z\",\n \"endTime\": \"2026-08-06T13:48:41.617977129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000880f00\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-pnf2fh5wjwql\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8925daa4-519a-42f1-a42f-eda89e216058\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8925daa4-519a-42f1-a42f-eda89e216058\",\n \"name\": \"8925daa4-519a-42f1-a42f-eda89e216058\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:29:35.264600406Z\",\n \"endTime\": \"2026-08-06T13:48:41.617977129Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f6f8c0ac957761ebc3c5fdef86a2b5100747f2089ab560235605af486280cd28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"a668f0b0a6511199b7c6d0a5e0fc0e43d56beafa0b386871867965a9f1ffc737","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-ntl57w\" with version 4.21.23 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9cd40\u003e: \n failed to create HCP cluster cluster-zstream-4-21-ntl57w: failed waiting for cluster=\"cluster-zstream-4-21-ntl57w\" in resourcegroup=\"rg-zstream-upgrade-4-21-tdr6mr4275wq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/00f2afbd-cf95-4467-a26f-0c9ca61e32c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/00f2afbd-cf95-4467-a26f-0c9ca61e32c3\",\n \"name\": \"00f2afbd-cf95-4467-a26f-0c9ca61e32c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:19:35.127061483Z\",\n \"endTime\": \"2026-08-06T13:38:40.310199162Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:110]: failed to create HCP cluster \"cluster-zstream-4-21-ntl57w\" with version 4.21.23 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9cd40\u003e: \n failed to create HCP cluster cluster-zstream-4-21-ntl57w: failed waiting for cluster=\"cluster-zstream-4-21-ntl57w\" in resourcegroup=\"rg-zstream-upgrade-4-21-tdr6mr4275wq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/00f2afbd-cf95-4467-a26f-0c9ca61e32c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/00f2afbd-cf95-4467-a26f-0c9ca61e32c3\",\n \"name\": \"00f2afbd-cf95-4467-a26f-0c9ca61e32c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:19:35.127061483Z\",\n \"endTime\": \"2026-08-06T13:38:40.310199162Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fb69549bab13ed6faa1b2767468c302498756540dd6f53149147a4389194052c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"14899c226e729562a794009d5069ba2a91359831b77c6dd029233b500a2d34ad","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001108210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dkkn6llwn9ml\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\",\n \"name\": \"6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:24:37.727370739Z\",\n \"endTime\": \"2026-08-06T13:48:41.816178395Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s139utb7gn0spngljdld5adcoi9lknv still exists (deletion dispatched at 2026-08-06T13:24:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n \u003c*errors.joinError | 0xc001108210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dkkn6llwn9ml\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\",\n \"name\": \"6a058d23-b76e-4230-8a1e-4b6d1a6bf1c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:24:37.727370739Z\",\n \"endTime\": \"2026-08-06T13:48:41.816178395Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s139utb7gn0spngljdld5adcoi9lknv still exists (deletion dispatched at 2026-08-06T13:24:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a895eaf5fc00535af027e673197376a531131993383b5ea3abe908eddc6e9ca7","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ff9aea087fe006f9e903db892300c98616e984acb196c62f9fe2abefab46b93b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085342109916729344","occurred_at":"2026-08-06T12:27:57Z","failed_at":"e2e","signature_id":"631aff46b4ed57edbe2f280d536a16b29329f90aacc852a48bdb580e6b43319f","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004aea40\u003e: \n failed to create HCP cluster ea-listhjmvfp: failed waiting for cluster=\"ea-listhjmvfp\" in resourcegroup=\"ea-list-vbbgk86jvtck\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/226ad230-01df-4367-9185-ab8b4bff928d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/226ad230-01df-4367-9185-ab8b4bff928d\",\n \"name\": \"226ad230-01df-4367-9185-ab8b4bff928d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:24:11.533675253Z\",\n \"endTime\": \"2026-08-06T13:43:30.417851569Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004aea40\u003e: \n failed to create HCP cluster ea-listhjmvfp: failed waiting for cluster=\"ea-listhjmvfp\" in resourcegroup=\"ea-list-vbbgk86jvtck\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/226ad230-01df-4367-9185-ab8b4bff928d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/226ad230-01df-4367-9185-ab8b4bff928d\",\n \"name\": \"226ad230-01df-4367-9185-ab8b4bff928d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-08-06T13:24:11.533675253Z\",\n \"endTime\": \"2026-08-06T13:43:30.417851569Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":29,"failure_pattern_ids":["0438873facb9dd98e74695c849423f31b775588ec5b0ef7ead10a8182c141d31","36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","a895eaf5fc00535af027e673197376a531131993383b5ea3abe908eddc6e9ca7","d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","dcbb98b30046e6cc7914c7e62bede6790189f033e9f8bb7bf919eebfa6754c56"],"matched_occurrences":29,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085318403903459328","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T10:53:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085287360039227392","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T08:50:24Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085286227883331584","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T08:45:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085284916701958144","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T08:40:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085238414642253824","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T05:35:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085234698753150976","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T05:21:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085233864640303104","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T05:17:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085231635954929664","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T05:08:58Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2085184094248898560","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T02:00:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2085182103493808128","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-06T01:52:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}