{"meta":{"date":"2026-07-30","timezone":"UTC","generated_at":"2026-08-26T13:12:31Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":20,"failed_runs":8,"runs_with_occurrences":8,"runs_with_failure_pattern_matches":7,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":12,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2082964456261816320","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-30T23:00:00Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9664715991840d15479b9ea975e91d94447304f7d07c52f77f570f3fe9f29859","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2082964456261816320","occurred_at":"2026-07-30T23:00:00Z","failed_at":"e2e","signature_id":"cc554efcb1bebe7d990883ab94bead887ef4c10bf0225aa8d891690288360dae","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004351e0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-q9gpdmtglj9j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q9gpdmtglj9j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004351e0\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-q9gpdmtglj9j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-q9gpdmtglj9j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"01f15bd280db5d89c938e3274f737786d9867d8b1ba2c2fbc749dba703089f61","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'uksouth' and API version '2026-06-30-preview' for type 'HCPOpenShiftClusters'. The supported api-versions are '2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'centraluseuap, eastus2euap, uksouth, switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["01f15bd280db5d89c938e3274f737786d9867d8b1ba2c2fbc749dba703089f61"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082935276501471232","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T21:04:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082914611102748672","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T19:41:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082899684623388672","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-30T18:42:37Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"7d6ca790bf7b051680c4c6291a7a27e82a7e0ae097e8f883bec6ce1d3e35fd81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082899684623388672","occurred_at":"2026-07-30T18:42:37Z","failed_at":"e2e","signature_id":"409703503e6aea948ba52c3c5eb1f2c0d8db728a3160f7b9df9d614dc003e6cc","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fb0258\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fb0258\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded"}},{"row_id":"88664c15a3df5ba26b0d9b8c090e40264ba461226ee430406a2b2c774a8020ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082899684623388672","occurred_at":"2026-07-30T18:42:37Z","failed_at":"e2e","signature_id":"7d2bcaa645afc4ae9592bf1820a2171f5d94eaca375fb1bd194afd110b7d5809","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000606880\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-7xkpn5bkr29m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/06b3812c-7ebf-4b90-8f77-3d0a9e26580c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/06b3812c-7ebf-4b90-8f77-3d0a9e26580c\",\n      \"name\": \"06b3812c-7ebf-4b90-8f77-3d0a9e26580c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T19:13:37.600674519Z\",\n      \"endTime\": \"2026-07-30T19:32:46.554485887Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000606880\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-7xkpn5bkr29m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/06b3812c-7ebf-4b90-8f77-3d0a9e26580c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/06b3812c-7ebf-4b90-8f77-3d0a9e26580c\",\n      \"name\": \"06b3812c-7ebf-4b90-8f77-3d0a9e26580c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T19:13:37.600674519Z\",\n      \"endTime\": \"2026-07-30T19:32:46.554485887Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082896908895916032","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T18:31:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082896087345008640","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T18:28:20Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082855991707176960","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-30T15:49:00Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8422a92dea9e725a19fce088b862b3a868939570627d22e73a547e278e2c7208","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082855991707176960","occurred_at":"2026-07-30T15:49:00Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082853583513980928","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T15:39:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082809676285612032","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-30T12:44:58Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"51d4cc1aa2bc5cea1d9574e28165ccddf94335efbd8866520b2f0c68bde6c891","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082809676285612032","occurred_at":"2026-07-30T12:44:58Z","failed_at":"e2e","signature_id":"b2240ed412ce4ac7629bcdff474ee13e9115652c8ff23691e840266cdb7e0f86","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:145]: failed to delete customer resource group \"cx-rg-cluster-pxnh4n9clmln\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001ade210\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-pxnh4n9clmln, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:145]: failed to delete customer resource group \"cx-rg-cluster-pxnh4n9clmln\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001ade210\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-pxnh4n9clmln, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ae6a7f7d2e3c3444ed20150a67378f751586e201de3a927f677f72a871b2bf44","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-pxnh4n9clmln, error: context deadline exceeded"}},{"row_id":"abd0ffb8c4ddd671e7ba01f9e8a1ef41dacb524a33c3b5a6d20b4c2c3cb7900a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082809676285612032","occurred_at":"2026-07-30T12:44:58Z","failed_at":"e2e","signature_id":"7a1811ca6cb73bd5963ea482dfbb3e212685b616d35f1d264dfe7bdc88eea013","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:411]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:410 with:\nAll ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]\nExpected\n    \u003c[]string | len:1, cap:1\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:411]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:410 with:\nAll ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]\nExpected\n    \u003c[]string | len:1, cap:1\u003e: ...\nto be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9095e6a1d01ab4218426574bc1b1f4094fa1640613eff3cdbf58f78fedc6a291","failure_pattern":"All ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]","search_query":"All ClusterOperators should report Available=True, but these are not available: [monitoring (Available=Unknown)]"}},{"row_id":"c68928a185abb05224fab9927820934f01bbe2f49cae83f3a21ac27577998b81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082809676285612032","occurred_at":"2026-07-30T12:44:58Z","failed_at":"e2e","signature_id":"1ab6fbef233dddf10f89d2a7fc671a3b1aaa7128d75d931d0bc6f823883a1e94","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-fk7h4x\" for cluster \"cluster-candidate-4-20-fk7h4x\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d80d60\u003e: \n    failed to create NodePool np-fk7h4x: failed waiting for nodepool=\"np-fk7h4x\" for cluster \"cluster-candidate-4-20-fk7h4x\" in resourcegroup=\"rg-candidate-4-20-87krhtz52qvw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/253ac28d-0828-4407-ba5f-731d8255e9b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/253ac28d-0828-4407-ba5f-731d8255e9b8\",\n      \"name\": \"253ac28d-0828-4407-ba5f-731d8255e9b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T13:18:30.696627194Z\",\n      \"endTime\": \"2026-07-30T13:37:34.938776476Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: failed to create node pool \"np-fk7h4x\" for cluster \"cluster-candidate-4-20-fk7h4x\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d80d60\u003e: \n    failed to create NodePool np-fk7h4x: failed waiting for nodepool=\"np-fk7h4x\" for cluster \"cluster-candidate-4-20-fk7h4x\" in resourcegroup=\"rg-candidate-4-20-87krhtz52qvw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/253ac28d-0828-4407-ba5f-731d8255e9b8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/253ac28d-0828-4407-ba5f-731d8255e9b8\",\n      \"name\": \"253ac28d-0828-4407-ba5f-731d8255e9b8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T13:18:30.696627194Z\",\n      \"endTime\": \"2026-07-30T13:37:34.938776476Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","9095e6a1d01ab4218426574bc1b1f4094fa1640613eff3cdbf58f78fedc6a291","ae6a7f7d2e3c3444ed20150a67378f751586e201de3a927f677f72a871b2bf44"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082802983245975552","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T12:18:22Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082801750829436928","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T12:13:23Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082801686253932544","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T12:13:04Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-30T10:01:16Z"},"failed_at":["e2e"],"failed_test_count":15,"occurrences":[{"row_id":"119ed4868599a7c5a7a0d067c26ac766ed20ce9307ffe6638928935b39359ac7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"39d0da823060c3256ca361233cddb71ef25ed0003a89f41041e8c0a328409a12","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:477]: failed to create nodepool nps-420-422 at version 4.20.31\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00059d820\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-h2zsf4\" in resourcegroup=\"rg-np-skip-minor-h2zsf4-b89k8srsvpkk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4149625e-4e5f-4a3a-bc28-d3fd3193e580\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4149625e-4e5f-4a3a-bc28-d3fd3193e580\",\n      \"name\": \"4149625e-4e5f-4a3a-bc28-d3fd3193e580\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:13:08.228285368Z\",\n      \"endTime\": \"2026-07-30T11:32:19.157857139Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:477]: failed to create nodepool nps-420-422 at version 4.20.31\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00059d820\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-h2zsf4\" in resourcegroup=\"rg-np-skip-minor-h2zsf4-b89k8srsvpkk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4149625e-4e5f-4a3a-bc28-d3fd3193e580\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4149625e-4e5f-4a3a-bc28-d3fd3193e580\",\n      \"name\": \"4149625e-4e5f-4a3a-bc28-d3fd3193e580\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:13:08.228285368Z\",\n      \"endTime\": \"2026-07-30T11:32:19.157857139Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5089ed72ca7a98aac379d293c4efbf4b3500b7e5a3ffb41b04168a7ec7a47871","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"b52edbdd27541815677acf3313e8a385ea427ab0cb29141a18288e99da41fcef","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:330]: failed to create nodepool npnoedge-4-20 at version 4.20.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001010980\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8fbqqx\" in resourcegroup=\"rg-np-noedge-8fbqqx-gpzkq677gbw9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/83da4bb2-8e81-42c2-bc1f-51306215a06d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/83da4bb2-8e81-42c2-bc1f-51306215a06d\",\n      \"name\": \"83da4bb2-8e81-42c2-bc1f-51306215a06d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:11:05.787386838Z\",\n      \"endTime\": \"2026-07-30T11:30:19.267053416Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:330]: failed to create nodepool npnoedge-4-20 at version 4.20.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001010980\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8fbqqx\" in resourcegroup=\"rg-np-noedge-8fbqqx-gpzkq677gbw9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/83da4bb2-8e81-42c2-bc1f-51306215a06d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/83da4bb2-8e81-42c2-bc1f-51306215a06d\",\n      \"name\": \"83da4bb2-8e81-42c2-bc1f-51306215a06d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:11:05.787386838Z\",\n      \"endTime\": \"2026-07-30T11:30:19.267053416Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"51e8e1c015ad306367a3ab01efa2b48dd36d84e6f661c6bf24e143730f6fe82a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"3f36455f13241999b39b257f5b49cb419277fb42d5614d28b4e671d9055c4d03","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:185]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00294ae10\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djtcdz2tpstd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-djtcdz2tpstd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:185]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00294ae10\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djtcdz2tpstd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-djtcdz2tpstd, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1141d0fd9f679007e447ee31146ba219b79d56c1f3a9288dd68187b3f9e7036c","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djtcdz2tpstd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-djtcdz2tpstd, error: context deadline exceeded"}},{"row_id":"5a66cdb885fcd9ba0f80ee79899b9ed7f8de2a3e3f243fb248f950b1593aa277","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"b89c7ac67f41ad70087c234e520a4a2da57fc04986a62c8408dfe0724611b225","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005a2c40\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-8mclbsqjtf6m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/321a074f-327c-4334-8a5c-1f095d42d969\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/321a074f-327c-4334-8a5c-1f095d42d969\",\n      \"name\": \"321a074f-327c-4334-8a5c-1f095d42d969\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:17:15.938139325Z\",\n      \"endTime\": \"2026-07-30T11:36:19.215744518Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005a2c40\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-8mclbsqjtf6m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/321a074f-327c-4334-8a5c-1f095d42d969\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/321a074f-327c-4334-8a5c-1f095d42d969\",\n      \"name\": \"321a074f-327c-4334-8a5c-1f095d42d969\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:17:15.938139325Z\",\n      \"endTime\": \"2026-07-30T11:36:19.215744518Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6b1c4865c1c2ecb293f86e26916d475ae5d7b1cbe59e591b148d6a8482574f70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"a7ae58b0c35201d9638220ac10a4f2fa47986d5db40a05cdc9220891d4f429e3","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001e86348\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"external-auth-cluster-55gxhxj67hwg\", caused by: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-55gxhxj67hwg\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-55gxhxj67hwg, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-55gxhxj67hwg\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-55gxhxj67hwg, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001e86348\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"external-auth-cluster-55gxhxj67hwg\", caused by: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-55gxhxj67hwg\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-55gxhxj67hwg, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-55gxhxj67hwg\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-55gxhxj67hwg, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded"}},{"row_id":"7855f439ffe9dd6a9f5d4066dfdf1a5944c77f6e59426c2f4494879982066928","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"6ac211ba10e267bd185623c33c1580de83901464fb1a55e5f6787ef19f4c86bf","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:219]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6f760\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mxv4cfnvmvfm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c4814543-e6f4-4086-8a84-c7857bb4b347\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c4814543-e6f4-4086-8a84-c7857bb4b347\",\n      \"name\": \"c4814543-e6f4-4086-8a84-c7857bb4b347\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:12:24.693112447Z\",\n      \"endTime\": \"2026-07-30T11:31:29.283599663Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:219]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f6f760\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-mxv4cfnvmvfm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c4814543-e6f4-4086-8a84-c7857bb4b347\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c4814543-e6f4-4086-8a84-c7857bb4b347\",\n      \"name\": \"c4814543-e6f4-4086-8a84-c7857bb4b347\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:12:24.693112447Z\",\n      \"endTime\": \"2026-07-30T11:31:29.283599663Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e12e4d3945f32fa65b823d7efbb5766f489a97645c2301085368818bc30b193","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"8ec59ee03d15896018b5346f71607e10a2e74944efd51bc21c29d9053c3d03d0","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0a980\u003e: \n    failed to create HCP cluster np-dg-minor-d2c552: failed waiting for cluster=\"np-dg-minor-d2c552\" in resourcegroup=\"rg-np-dg-minor-d2c552-s6hmsbqstnwg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb7b0509-b11a-4301-9d26-8282b9e6c9e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb7b0509-b11a-4301-9d26-8282b9e6c9e3\",\n      \"name\": \"eb7b0509-b11a-4301-9d26-8282b9e6c9e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:13:06.699460126Z\",\n      \"endTime\": \"2026-07-30T11:32:19.320944112Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:736]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b0a980\u003e: \n    failed to create HCP cluster np-dg-minor-d2c552: failed waiting for cluster=\"np-dg-minor-d2c552\" in resourcegroup=\"rg-np-dg-minor-d2c552-s6hmsbqstnwg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb7b0509-b11a-4301-9d26-8282b9e6c9e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/eb7b0509-b11a-4301-9d26-8282b9e6c9e3\",\n      \"name\": \"eb7b0509-b11a-4301-9d26-8282b9e6c9e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:13:06.699460126Z\",\n      \"endTime\": \"2026-07-30T11:32:19.320944112Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9e1e0fb8ef057c9b4a73b57fc8de04632a22f9fe304316d5495bc63667cb9217","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"b092477ead04bebcbc3e54ba7df3b6a17e380d8467de8e0d18c3adfe414687c1","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-21 to version 4.21.26\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001096720\u003e: \n    failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-m98bwq\" resourcegroup=\"rg-np-version-upgrade-m98bwq-6wlkt7dwt5st\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-m98bwq in resource group rg-np-version-upgrade-m98bwq-6wlkt7dwt5st, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-21 to version 4.21.26\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001096720\u003e: \n    failed waiting for nodepool=\"npupgrade-4-21\" in cluster=\"np-version-upgrade-cluster-m98bwq\" resourcegroup=\"rg-np-version-upgrade-m98bwq-6wlkt7dwt5st\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-21 in cluster np-version-upgrade-cluster-m98bwq in resource group rg-np-version-upgrade-m98bwq-6wlkt7dwt5st, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1141d0fd9f679007e447ee31146ba219b79d56c1f3a9288dd68187b3f9e7036c","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djtcdz2tpstd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-djtcdz2tpstd, error: context deadline exceeded"}},{"row_id":"a1b71e3f82f03fb2e0586a571f037bbdb039aa0c966797440920fb9fd5d8f6d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"29160597ae1b571b364c2eeabf2769302264c37cc8a76086b92bf9b2a5805197","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a99b20\u003e: \n    failed to create HCP cluster np-downgrade-cp7c49: failed waiting for cluster=\"np-downgrade-cp7c49\" in resourcegroup=\"rg-np-downgrade-cp7c49-jvr6w2dtdt2g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b9dd0096-3c57-462e-a4e7-d5aae3bc006d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b9dd0096-3c57-462e-a4e7-d5aae3bc006d\",\n      \"name\": \"b9dd0096-3c57-462e-a4e7-d5aae3bc006d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:10:46.754483041Z\",\n      \"endTime\": \"2026-07-30T11:29:49.202419101Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a99b20\u003e: \n    failed to create HCP cluster np-downgrade-cp7c49: failed waiting for cluster=\"np-downgrade-cp7c49\" in resourcegroup=\"rg-np-downgrade-cp7c49-jvr6w2dtdt2g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b9dd0096-3c57-462e-a4e7-d5aae3bc006d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b9dd0096-3c57-462e-a4e7-d5aae3bc006d\",\n      \"name\": \"b9dd0096-3c57-462e-a4e7-d5aae3bc006d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:10:46.754483041Z\",\n      \"endTime\": \"2026-07-30T11:29:49.202419101Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a9c1583eb59f47e9a19a6f1fcf7f45fc3dc2fe2ee7a85eb051fa4861005c84c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"c23160afc432e787bd9c17aa2fcc0f9f45b004d10323c162c9f24a2573785276","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000290558\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cp-ystream-upgrade-4-21-dbzwqxwppnjs\", caused by: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-bn6vtx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-dbzwqxwppnjs\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cp-ystream-upgrade-4-21-bn6vtx in resource group rg-cp-ystream-upgrade-4-21-dbzwqxwppnjs, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-bn6vtx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-dbzwqxwppnjs\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cp-ystream-upgrade-4-21-bn6vtx in resource group rg-cp-ystream-upgrade-4-21-dbzwqxwppnjs, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000290558\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cp-ystream-upgrade-4-21-dbzwqxwppnjs\", caused by: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-bn6vtx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-dbzwqxwppnjs\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cp-ystream-upgrade-4-21-bn6vtx in resource group rg-cp-ystream-upgrade-4-21-dbzwqxwppnjs, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-bn6vtx\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-dbzwqxwppnjs\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cp-ystream-upgrade-4-21-bn6vtx in resource group rg-cp-ystream-upgrade-4-21-dbzwqxwppnjs, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded"}},{"row_id":"c14c8d3a22e3b19815e137c59be1a603a9e441e8e7b1689df22dd8662ea40a36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"6f5f5908ee3b39d280f7e0e40faf8eaef29dd4317f23980e3578da9280e33c74","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.21.26\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a941e0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-d45wt7\" resourcegroup=\"rg-np-version-upgrade-d45wt7-r5nbl95ln47b\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-d45wt7 in resource group rg-np-version-upgrade-d45wt7-r5nbl95ln47b, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.21.26\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001a941e0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-d45wt7\" resourcegroup=\"rg-np-version-upgrade-d45wt7-r5nbl95ln47b\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-d45wt7 in resource group rg-np-version-upgrade-d45wt7-r5nbl95ln47b, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1141d0fd9f679007e447ee31146ba219b79d56c1f3a9288dd68187b3f9e7036c","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djtcdz2tpstd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-djtcdz2tpstd, error: context deadline exceeded"}},{"row_id":"d5470878c05f62d71094a05e0182c5563d1f6685f4af52f5335eb6bcd338d346","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"f329a6520f95944717d8a48e410a87d34976660aacc46f3d9e2e9552341a68de","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e269a0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-k2rwpg65mjh7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/db817875-2c58-4fcf-8b75-f06b629d5f70\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/db817875-2c58-4fcf-8b75-f06b629d5f70\",\n      \"name\": \"db817875-2c58-4fcf-8b75-f06b629d5f70\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:15:52.679864378Z\",\n      \"endTime\": \"2026-07-30T11:34:59.507212138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:269]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e269a0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-k2rwpg65mjh7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/db817875-2c58-4fcf-8b75-f06b629d5f70\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/db817875-2c58-4fcf-8b75-f06b629d5f70\",\n      \"name\": \"db817875-2c58-4fcf-8b75-f06b629d5f70\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:15:52.679864378Z\",\n      \"endTime\": \"2026-07-30T11:34:59.507212138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f6c6cf6a170b07cd018fc5d8b35e9d5e9299995a4ddba26cf76e046dfab5da2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"e4a8538450a0681886da6c98e27e84371e6ba2092979838ec812325cc8162e03","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.20.31\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008aacf0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-dch7fj\" resourcegroup=\"rg-np-version-upgrade-dch7fj-w9xwlnksswtg\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-dch7fj in resource group rg-np-version-upgrade-dch7fj-w9xwlnksswtg, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:210]: failed to upgrade node pool npupgrade-4-20 to version 4.20.31\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008aacf0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-dch7fj\" resourcegroup=\"rg-np-version-upgrade-dch7fj-w9xwlnksswtg\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-dch7fj in resource group rg-np-version-upgrade-dch7fj-w9xwlnksswtg, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1141d0fd9f679007e447ee31146ba219b79d56c1f3a9288dd68187b3f9e7036c","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-djtcdz2tpstd\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-djtcdz2tpstd, error: context deadline exceeded"}},{"row_id":"fd9c7c2c9682b3544f4dda423db9cde72c56e628e026ee95bcb1f4a84b726006","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"2c788d8a3815e3876197d8f2e7d6ddac61b59269c848b674bc458f1031ac1846","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001168180\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-lq29kqkt9n82\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-lq29kqkt9n82\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-lq29kqkt9n82, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-lq29kqkt9n82\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-lq29kqkt9n82, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001168180\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-lq29kqkt9n82\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-lq29kqkt9n82\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-lq29kqkt9n82, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-lq29kqkt9n82\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-lq29kqkt9n82, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-7wqzzv\" in resourcegroup=\"admin-credential-lifecycle-test-9vnswkxphvxb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-7wqzzv in resource group admin-credential-lifecycle-test-9vnswkxphvxb, error: context deadline exceeded"}},{"row_id":"fe7bec9733f559b3d539cfeadef9db2e85134aabf0976da1847ca2631fd3dc4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082768480108548096","occurred_at":"2026-07-30T10:01:16Z","failed_at":"e2e","signature_id":"53759798de1b1cbea6d270261d69b47e856f4593abcfe1c2d0bc5850c3342479","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f2580\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-v96q89jj47wk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72daab77-8240-4076-a664-61b47746481d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72daab77-8240-4076-a664-61b47746481d\",\n      \"name\": \"72daab77-8240-4076-a664-61b47746481d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:14:44.528119762Z\",\n      \"endTime\": \"2026-07-30T11:33:49.266559979Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010f2580\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-v96q89jj47wk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72daab77-8240-4076-a664-61b47746481d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/72daab77-8240-4076-a664-61b47746481d\",\n      \"name\": \"72daab77-8240-4076-a664-61b47746481d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T11:14:44.528119762Z\",\n      \"endTime\": \"2026-07-30T11:33:49.266559979Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["1141d0fd9f679007e447ee31146ba219b79d56c1f3a9288dd68187b3f9e7036c","36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082754944435228672","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T09:07:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082706035193155584","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T05:53:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082675949039521792","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-30T03:53:35Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ae3ac4a8b3bdd3785791a1516591b7facf69b6765d119dd561c6fc929995c25e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082675949039521792","occurred_at":"2026-07-30T03:53:35Z","failed_at":"e2e","signature_id":"5b02f0fbf54d86701870a336fa2e752001e95ededdf41894d945e1f818c12826","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b1ede0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-s9ztsrg6s8z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c91435db-9a94-4c69-97ad-29629546f565\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c91435db-9a94-4c69-97ad-29629546f565\",\n      \"name\": \"c91435db-9a94-4c69-97ad-29629546f565\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T04:53:57.85626036Z\",\n      \"endTime\": \"2026-07-30T05:13:12.75339723Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:109]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b1ede0\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-s9ztsrg6s8z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c91435db-9a94-4c69-97ad-29629546f565\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/c91435db-9a94-4c69-97ad-29629546f565\",\n      \"name\": \"c91435db-9a94-4c69-97ad-29629546f565\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-30T04:53:57.85626036Z\",\n      \"endTime\": \"2026-07-30T05:13:12.75339723Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2082647614926360576","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-30T02:00:59Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"58a1321e6fe817eecc7e8fc8ad05ddacd28fd4b1e0bead75f6f182af33e5910d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2082647614926360576","occurred_at":"2026-07-30T02:00:59Z","failed_at":"e2e","signature_id":"e3f0e8d101926d54b52273c737c587f0ca58a59d12fbb0b58cb0f281747a8b4f","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e79e40\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e79e40\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"55a7042db8a639148ac918b05723e7c6b738fd94bd7db60b837e426029851f89","failure_pattern":"failed to create service principal: create service principal: error status code received from the API","search_query":"failed to create service principal: create service principal: error status code received from the API"}},{"row_id":"e6b7f3a38a96fbc0b02fc47045316274c86a27acfc4371eb3c1e0927834a0683","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2082647614926360576","occurred_at":"2026-07-30T02:00:59Z","failed_at":"e2e","signature_id":"664e785e76444d94ea714dbd3345cd8c8aa34df77b19996aacc8a85ca28e6c7e","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca1f40\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca1f40\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"55a7042db8a639148ac918b05723e7c6b738fd94bd7db60b837e426029851f89","failure_pattern":"failed to create service principal: create service principal: error status code received from the API","search_query":"failed to create service principal: create service principal: error status code received from the API"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["55a7042db8a639148ac918b05723e7c6b738fd94bd7db60b837e426029851f89"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082641152187568128","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-30T01:34:23Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1d209fc16e1b2b64f66dcc39e5d7965f1da6c9ac0803f4971b4471e9ec93cdbd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082641152187568128","occurred_at":"2026-07-30T01:34:23Z","failed_at":"e2e","signature_id":"0d1a1089eb774ffa3548638344a1e2b0fe4f14bbefe85bfa532befdca533fb72","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-49z6mq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004ae280\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-sncqc.apps.aro.b8c0k3m1i2y1q5p.hsnv.eastus2.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:186]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-49z6mq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0004ae280\u003e: \n    [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-sncqc.apps.aro.b8c0k3m1i2y1q5p.hsnv.eastus2.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082619116270653440","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T00:07:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082618608495628288","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-30T00:05:44Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}