{"meta":{"date":"2026-07-29","timezone":"UTC","generated_at":"2026-08-26T13:51:16Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":15,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":9,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2082602235849609216","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-29T23:00:40Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4b0c3c142007ae035572f41a39b1b311ac5738bc065ed8f7b25db238bf4caf4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2082602235849609216","occurred_at":"2026-07-29T23:00:40Z","failed_at":"e2e","signature_id":"a6b013ff1d8bdd34482eff2d5f6a091650cd8c4a6b2f95151702b83e709e2576","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e722c0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:127]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e722c0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"55a7042db8a639148ac918b05723e7c6b738fd94bd7db60b837e426029851f89","failure_pattern":"failed to create service principal: create service principal: error status code received from the API","search_query":"failed to create service principal: create service principal: error status code received from the API"}},{"row_id":"9174a81511550ff7d7afededc6a13e858d56a535ccb4d061e2c69bec6db58735","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2082602235849609216","occurred_at":"2026-07-29T23:00:40Z","failed_at":"e2e","signature_id":"3d8c987c527570b02284ae5b1dd3553b6cc602b543deeb4c7abc806b1b8eb8c9","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d25ea0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:264]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d25ea0\u003e: \n    failed to create service principal: create service principal: error status code received from the API\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"55a7042db8a639148ac918b05723e7c6b738fd94bd7db60b837e426029851f89","failure_pattern":"failed to create service principal: create service principal: error status code received from the API","search_query":"failed to create service principal: create service principal: error status code received from the API"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["55a7042db8a639148ac918b05723e7c6b738fd94bd7db60b837e426029851f89"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082572874022916096","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-29T21:04:00Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-29T18:28:59Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"357d66b6214f8162bbe55bd7f77e691eb9037d5477e482517c24de4111a0412c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","failed_at":"e2e","signature_id":"7e2a79de725776165b8aa00972dd0a55650a63cb3758764c653899700a45bcc1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00130a258\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\", caused by: failed waiting for hcpCluster=\"cluster-candidate-4-22-qwv8sf\" in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-qwv8sf in resource group rg-candidate-4-22-5pp8d59rjhfd, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-4-22-qwv8sf\" in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-qwv8sf in resource group rg-candidate-4-22-5pp8d59rjhfd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc00130a258\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\", caused by: failed waiting for hcpCluster=\"cluster-candidate-4-22-qwv8sf\" in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-qwv8sf in resource group rg-candidate-4-22-5pp8d59rjhfd, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-candidate-4-22-qwv8sf\" in resourcegroup=\"rg-candidate-4-22-5pp8d59rjhfd\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-candidate-4-22-qwv8sf in resource group rg-candidate-4-22-5pp8d59rjhfd, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded"}},{"row_id":"574557fce4a6de9efb1aca1c592f67ac948d72f09a5f11fb956ca624338c0b60","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","failed_at":"e2e","signature_id":"a17e2efd2f02a984b507dcc724ffb9c4f854ad0a7bf6c1ddd6785c1d3ad1414a","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00291bce0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:269]: failed to add password to app registration\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00291bce0\u003e: \n    add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"63a755b10a4ebc8c89991a40e36c8d3c0a9cd5e114a858de40038cb6cec08053","failure_pattern":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 1 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API"}},{"row_id":"9950a8a7eb5727292bd48f33e62f440e0bc519370952f9a82c82200adec2114c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","failed_at":"e2e","signature_id":"dbef0570edb7869dcc70fa2607e08962e0ef53d745127ecc46f504d82360bd8b","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e48150\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\", caused by: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-hxfs6rqqk2l8, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-hxfs6rqqk2l8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc000e48150\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\", caused by: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-hxfs6rqqk2l8, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hxfs6rqqk2l8\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-hxfs6rqqk2l8, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded"}},{"row_id":"d002ffbdca99ccd245035949cfef8a195e9d3f0e81e2d900a50c96027a74a88f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082533863174705152","occurred_at":"2026-07-29T18:28:59Z","failed_at":"e2e","signature_id":"bc1dfc2c55c0166aaf97f354cd93cc3c56076c1026d4eee5e529342b2fbe5534","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a2c0a8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a2c0a8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["63a755b10a4ebc8c89991a40e36c8d3c0a9cd5e114a858de40038cb6cec08053","a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082530876830257152","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-29T18:17:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082530217577943040","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-29T18:14:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082529674914697216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-29T18:12:20Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082484659781898240","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-29T15:08:42Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082426114898137088","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-29T11:20:50Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"55f73085136e132d6ab4ed98537ede77a9c3027869c51f2b9afcbc6a0977ab19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082426114898137088","occurred_at":"2026-07-29T11:20:50Z","failed_at":"e2e","signature_id":"fdc5d2be0d25f697e55b562829775e172533b19f5057a9135b273313e4c97e0b","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002462078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gv7f9fvg24h7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1465f2b7-2fd3-4347-8208-672d33732fee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1465f2b7-2fd3-4347-8208-672d33732fee\",\n      \"name\": \"1465f2b7-2fd3-4347-8208-672d33732fee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T11:50:37.856046587Z\",\n      \"endTime\": \"2026-07-29T12:09:40.769017857Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002462078\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gv7f9fvg24h7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1465f2b7-2fd3-4347-8208-672d33732fee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1465f2b7-2fd3-4347-8208-672d33732fee\",\n      \"name\": \"1465f2b7-2fd3-4347-8208-672d33732fee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T11:50:37.856046587Z\",\n      \"endTime\": \"2026-07-29T12:09:40.769017857Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36922e7c617dedf0757391f782f539692708bab811bdd89eae5827b4dc3ffe03"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082380610684325888","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-29T08:20:00Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082332246349451264","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-29T05:07:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330659770077184","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-29T05:01:31Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-29T04:59:16Z"},"failed_at":["e2e"],"failed_test_count":8,"occurrences":[{"row_id":"3cfa29004990915e39c9eb9138b599c7ce7dfcebcbcb8b5b3692188eca9cd2d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","failed_at":"e2e","signature_id":"69bd66b6d04fbda55431cca5ce2192616e394de058d01d12e657d8cad6c4e68d","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:219]: Timed out after 2520.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:215 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"8a30b5a7171debf3b781ff8f0c4b4b631408c8dc3da1b684e485c4faf74908ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","failed_at":"e2e","signature_id":"3320b7b421abc7495f3d38d4c54d92df2ee5569a5b7507340497c93c97a16b2b","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e44d60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8t8h2tgjbrcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\",\n      \"name\": \"e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:32.496403767Z\",\n      \"endTime\": \"2026-07-29T05:33:46.625119368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:79]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e44d60\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8t8h2tgjbrcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\",\n      \"name\": \"e7fe9e69-daaa-4497-9a2d-77e15fd09b7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:32.496403767Z\",\n      \"endTime\": \"2026-07-29T05:33:46.625119368Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"91e385284d815e2e3c54666a0636b93e1976fd231b8d4df03ca558312f138db1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","failed_at":"e2e","signature_id":"eee34cb496c5a8c7be29d16abaf5d0a666a957636e08b1c93061315751f8b52f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-20-7kl5nw54qr69/cluster-candidate-4-20-znj4vt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a944e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-znj4vt: failed waiting for cluster=\"cluster-candidate-4-20-znj4vt\" in resourcegroup=\"rg-candidate-4-20-7kl5nw54qr69\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d9d17143-8160-42f2-ba83-7728bde83b14\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d9d17143-8160-42f2-ba83-7728bde83b14\",\n      \"name\": \"d9d17143-8160-42f2-ba83-7728bde83b14\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:32.584383772Z\",\n      \"endTime\": \"2026-07-29T05:33:46.687822425Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-20-7kl5nw54qr69/cluster-candidate-4-20-znj4vt should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a944e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-znj4vt: failed waiting for cluster=\"cluster-candidate-4-20-znj4vt\" in resourcegroup=\"rg-candidate-4-20-7kl5nw54qr69\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d9d17143-8160-42f2-ba83-7728bde83b14\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/d9d17143-8160-42f2-ba83-7728bde83b14\",\n      \"name\": \"d9d17143-8160-42f2-ba83-7728bde83b14\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:32.584383772Z\",\n      \"endTime\": \"2026-07-29T05:33:46.687822425Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9da088c539022001a75a3d31bae6a09c6ca6f67283bac78b85d2d5aab450b96b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","failed_at":"e2e","signature_id":"985863ee16558d554939b5436eded2bc5d8e28deece7209755185db265ecdbd2","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00071afc0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/4772936f-ae74-485e-9e7e-40a7043cde4c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/4772936f-ae74-485e-9e7e-40a7043cde4c\",\n      \"name\": \"4772936f-ae74-485e-9e7e-40a7043cde4c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:15:27.955090377Z\",\n      \"endTime\": \"2026-07-29T05:34:46.282514393Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00071afc0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/4772936f-ae74-485e-9e7e-40a7043cde4c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/4772936f-ae74-485e-9e7e-40a7043cde4c\",\n      \"name\": \"4772936f-ae74-485e-9e7e-40a7043cde4c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:15:27.955090377Z\",\n      \"endTime\": \"2026-07-29T05:34:46.282514393Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b0b4e81f6df5e9073864839e6e0fd3101f21acd3233f4ed403f4de1a965fcd21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","failed_at":"e2e","signature_id":"89ec90a05854a96e4314d56d7b2103aca5da8c5827d4d0912eba0a8b49920eb7","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001456540\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-ghbtv2frk8jv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/07ea6a1c-0e10-4d23-bee2-ed41df029a81\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/07ea6a1c-0e10-4d23-bee2-ed41df029a81\",\n      \"name\": \"07ea6a1c-0e10-4d23-bee2-ed41df029a81\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:31.670658764Z\",\n      \"endTime\": \"2026-07-29T05:33:46.645534989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001456540\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-ghbtv2frk8jv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/07ea6a1c-0e10-4d23-bee2-ed41df029a81\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/07ea6a1c-0e10-4d23-bee2-ed41df029a81\",\n      \"name\": \"07ea6a1c-0e10-4d23-bee2-ed41df029a81\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:31.670658764Z\",\n      \"endTime\": \"2026-07-29T05:33:46.645534989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b2b8468423f771b948080eb696b19b147fe20fdd66c5596a55406afd4c344f9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","failed_at":"e2e","signature_id":"bf6b529a259dfb87e6ef3b60ae798f6e0f770f84da51d8163a2c8622149afbd5","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef86c0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-krvt2jsl464l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e72a9dba-3b99-45a7-a7d5-09149827f4b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e72a9dba-3b99-45a7-a7d5-09149827f4b7\",\n      \"name\": \"e72a9dba-3b99-45a7-a7d5-09149827f4b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:32.010755493Z\",\n      \"endTime\": \"2026-07-29T05:33:46.238454371Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:87]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ef86c0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-krvt2jsl464l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e72a9dba-3b99-45a7-a7d5-09149827f4b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e72a9dba-3b99-45a7-a7d5-09149827f4b7\",\n      \"name\": \"e72a9dba-3b99-45a7-a7d5-09149827f4b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:32.010755493Z\",\n      \"endTime\": \"2026-07-29T05:33:46.238454371Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cef17545b8c32914c2941ba4612b0f354316314ec63f79c1e26dfe46e9ae16dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","failed_at":"e2e","signature_id":"c46ba0cb07eeb66c683f5e69d56a31d8f06a3223bda86d64c6cbafa4257b8b22","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-21-p4kjqctgmzk6/cluster-candidate-4-21-glndl7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000644800\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-glndl7: failed waiting for cluster=\"cluster-candidate-4-21-glndl7\" in resourcegroup=\"rg-candidate-4-21-p4kjqctgmzk6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e586cdc4-e3f0-4000-a84e-66355c4b12ff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e586cdc4-e3f0-4000-a84e-66355c4b12ff\",\n      \"name\": \"e586cdc4-e3f0-4000-a84e-66355c4b12ff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:17:47.986658491Z\",\n      \"endTime\": \"2026-07-29T05:37:06.506828647Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-21-p4kjqctgmzk6/cluster-candidate-4-21-glndl7 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000644800\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-glndl7: failed waiting for cluster=\"cluster-candidate-4-21-glndl7\" in resourcegroup=\"rg-candidate-4-21-p4kjqctgmzk6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e586cdc4-e3f0-4000-a84e-66355c4b12ff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/e586cdc4-e3f0-4000-a84e-66355c4b12ff\",\n      \"name\": \"e586cdc4-e3f0-4000-a84e-66355c4b12ff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:17:47.986658491Z\",\n      \"endTime\": \"2026-07-29T05:37:06.506828647Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ec810f25600d9ecaf78ff67fd912a2e9244bfed80bb680adccf9636c5b0db005","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082330094562447360","occurred_at":"2026-07-29T04:59:16Z","failed_at":"e2e","signature_id":"83a466dabf4df209fbbc153c92389d7b33d53abfa5d0ea52538017c262aa8d7d","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006a2180\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jp5gqq6r46ph\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\",\n      \"name\": \"f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:31.662472487Z\",\n      \"endTime\": \"2026-07-29T05:33:46.618172895Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006a2180\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jp5gqq6r46ph\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\",\n      \"name\": \"f30d0249-fe7e-4dbb-ab0e-f68c73be2bba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T05:14:31.662472487Z\",\n      \"endTime\": \"2026-07-29T05:33:46.618172895Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082288976865005568","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-29T02:15:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2082285143296315392","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-29T02:00:39Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2e64d0129f71f1dfcbaba1dd01098367318108660ac75693e4a6304175191eda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2082285143296315392","occurred_at":"2026-07-29T02:00:39Z","failed_at":"e2e","signature_id":"fe1ab3d15d93157c8f567e9b49bfe94cb77c39e9d6c00f3889916cd9f2f74c0b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-21-fvwltc4gz4f6/cluster-candidate-4-21-49lzb5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b2d780\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-49lzb5: failed waiting for cluster=\"cluster-candidate-4-21-49lzb5\" in resourcegroup=\"rg-candidate-4-21-fvwltc4gz4f6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1d29f20f-5fbd-42e5-87c0-2a58dcea72d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1d29f20f-5fbd-42e5-87c0-2a58dcea72d4\",\n      \"name\": \"1d29f20f-5fbd-42e5-87c0-2a58dcea72d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T02:06:05.891858654Z\",\n      \"endTime\": \"2026-07-29T02:25:10.15756699Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: HCP cluster rg-candidate-4-21-fvwltc4gz4f6/cluster-candidate-4-21-49lzb5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b2d780\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-49lzb5: failed waiting for cluster=\"cluster-candidate-4-21-49lzb5\" in resourcegroup=\"rg-candidate-4-21-fvwltc4gz4f6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1d29f20f-5fbd-42e5-87c0-2a58dcea72d4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1d29f20f-5fbd-42e5-87c0-2a58dcea72d4\",\n      \"name\": \"1d29f20f-5fbd-42e5-87c0-2a58dcea72d4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-07-29T02:06:05.891858654Z\",\n      \"endTime\": \"2026-07-29T02:25:10.15756699Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082259111117852672","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-29T00:17:13Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"33c8892b68eaf39addb4c2661486377717027100998dd3cf662f22eb2acf6aa9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2082259111117852672","occurred_at":"2026-07-29T00:17:13Z","failed_at":"e2e","signature_id":"55c6bf9689c47e7e895ffff2e4c5a16c064dfdccbb11b42f90d9359f0f01c1f9","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016ec330\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"idms-jn69xnhg7z62\", caused by: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-jn69xnhg7z62\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-jn69xnhg7z62, error: context deadline exceeded, error: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-jn69xnhg7z62\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-jn69xnhg7z62, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:285]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016ec330\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"idms-jn69xnhg7z62\", caused by: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-jn69xnhg7z62\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-jn69xnhg7z62, error: context deadline exceeded, error: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-jn69xnhg7z62\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-jn69xnhg7z62, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-82hd5nnf8hsk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-82hd5nnf8hsk, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a0f7097119093087677b064ee2bb11f639db773e33b710f4fbee9a6c1480fc09"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}