{"meta":{"date":"2026-10-08","timezone":"UTC","generated_at":"2026-10-29T14:10:32Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108334615943974912","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T23:11:58Z","started_at":"2026-10-08T23:11:58Z","completed_at":"2026-10-09T01:23:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-09T01:41:40.048555182Z","timing_metadata_checked_at":"2026-10-09T01:41:40.048555182Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-09T01:41:40.056884355Z","region_metadata_checked_at":"2026-10-09T01:41:40.056884355Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108289457667969024","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T20:12:31Z","started_at":"2026-10-08T20:12:31Z","completed_at":"2026-10-08T23:04:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T23:07:02.586049207Z","timing_metadata_checked_at":"2026-10-08T23:07:02.586049207Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T23:07:02.589188598Z","region_metadata_checked_at":"2026-10-08T23:07:02.589188598Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"33322c9eecac71a00d5695835c0886c0058558f83c914f3b48ee334d81841da0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108289457667969024","occurred_at":"2026-10-08T20:12:31Z","failed_at":"e2e","signature_id":"4a13b0e3b632754c621cd1f40992e4f49a86b9b5a5adaf7a0291091d3c763eaa","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x25f97d5e2360\u003e: \n    resource group \"tls-endpoint-cluster-z5gqw56sgjhd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-z5gqw56sgjhd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n      \"name\": \"cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T21:40:36.777928134Z\",\n      \"endTime\": \"2026-10-08T22:04:36.806096142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tarq2rh1nt9l8n2qiphckhb46c0kvk6 still exists (deletion dispatched at 2026-10-08T21:40:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x25f97d5e2360\u003e: \n    resource group \"tls-endpoint-cluster-z5gqw56sgjhd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-z5gqw56sgjhd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n      \"name\": \"cc8151c4-e387-49ba-89c6-800d9fe39b7c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T21:40:36.777928134Z\",\n      \"endTime\": \"2026-10-08T22:04:36.806096142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tarq2rh1nt9l8n2qiphckhb46c0kvk6 still exists (deletion dispatched at 2026-10-08T21:40:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108259102915301376","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T18:11:54Z","started_at":"2026-10-08T18:11:54Z","completed_at":"2026-10-08T21:06:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T21:11:10.594225277Z","timing_metadata_checked_at":"2026-10-08T21:11:10.594225277Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T21:11:10.598475065Z","region_metadata_checked_at":"2026-10-08T21:11:10.598475065Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T15:43:42Z","started_at":"2026-10-08T15:43:42Z","completed_at":"2026-10-08T18:58:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T19:03:59.473658863Z","timing_metadata_checked_at":"2026-10-08T19:03:59.473658863Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T19:03:59.478913946Z","region_metadata_checked_at":"2026-10-08T19:03:59.478913946Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"15f6518f2bf61dfd3ea93b0c5de1b280d5a636d81d3718e17d5b6f3a4af16ed9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","failed_at":"e2e","signature_id":"c37035f44e6baf132cd362aa9454d42aa8651a4d8c8d859449fd5863866a8dc3","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:148]: failed to delete customer resource group \"cx-rg-cluster-9528856dtms2\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2df35190540\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\n    {\n        msg: \"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2df35ff14c0\u003e{\n                s: \"timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:148]: failed to delete customer resource group \"cx-rg-cluster-9528856dtms2\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2df35190540\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\n    {\n        msg: \"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2df35ff14c0\u003e{\n                s: \"timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8a28d48cc4970d0f2235caa51b21154be46f90f637aecb0006fe72ad7a4ea5","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-9528856dtms2, error: context deadline exceeded"}},{"row_id":"3c9b4187a3ac94829ee48978334868fc49174537d57315f5c6a841bb63aa59df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","failed_at":"e2e","signature_id":"7801a21b4f98263d138c84dd4d41be41bec31d075db9a5534f6fe7a256d4fe97","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x317d01c40258\u003e: \n    resource group \"v20260901preview-9cxtwb94v2tg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-9cxtwb94v2tg\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n      \"name\": \"4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:03:49.176345278Z\",\n      \"endTime\": \"2026-10-08T17:27:51.502199148Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmtqk63qrfodamveh24r6lp7p43kt still exists (deletion dispatched at 2026-10-08T17:03:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x317d01c40258\u003e: \n    resource group \"v20260901preview-9cxtwb94v2tg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-9cxtwb94v2tg\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n      \"name\": \"4816c7c1-1d1e-40dc-9966-7fde6d93956a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:03:49.176345278Z\",\n      \"endTime\": \"2026-10-08T17:27:51.502199148Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmtqk63qrfodamveh24r6lp7p43kt still exists (deletion dispatched at 2026-10-08T17:03:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7ce576e5622c2cd15b9fa8b19a44b4c91c456739b9a616c0260a5c3dbd1524dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","failed_at":"e2e","signature_id":"afaf794a0f74bf5028bd0c8acfda702e8fa09c3b57b33581a270af808e618e1a","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x19f9b5e0a3f0\u003e: \n    resource group \"customer-rg-wkldzssbjxmj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-wkldzssbjxmj\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\",\n      \"name\": \"eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:08:36.973340805Z\",\n      \"endTime\": \"2026-10-08T17:32:41.526425453Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tannaefijum1ouk23vffl0tmfuhorgq still exists (deletion dispatched at 2026-10-08T17:08:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x19f9b5e0a3f0\u003e: \n    resource group \"customer-rg-wkldzssbjxmj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-wkldzssbjxmj\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\",\n      \"name\": \"eb82b4ed-34b1-4bae-b90f-ec9ebb76c7a8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:08:36.973340805Z\",\n      \"endTime\": \"2026-10-08T17:32:41.526425453Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tannaefijum1ouk23vffl0tmfuhorgq still exists (deletion dispatched at 2026-10-08T17:08:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b202cd2ceed97d85f49097aa18f27a043a6ccabc399c0010d0511712d6e27c91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108221811563958272","occurred_at":"2026-10-08T15:43:42Z","failed_at":"e2e","signature_id":"2bccea3f5889cc8d014f896b80d99afeb5d988c9e408682319482453975203f2","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x63eda6241c8\u003e: \n    resource group \"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n      \"name\": \"c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:03:08.268369953Z\",\n      \"endTime\": \"2026-10-08T17:27:11.496076999Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmuj8loas31r4p690qgf675h7c53q still exists (deletion dispatched at 2026-10-08T17:03:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x63eda6241c8\u003e: \n    resource group \"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-sppg9t7zzmdm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n      \"name\": \"c3f349e3-c307-4eb6-8abb-0bf949fe42f5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T17:03:08.268369953Z\",\n      \"endTime\": \"2026-10-08T17:27:11.496076999Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2tanmuj8loas31r4p690qgf675h7c53q still exists (deletion dispatched at 2026-10-08T17:03:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["6d8a28d48cc4970d0f2235caa51b21154be46f90f637aecb0006fe72ad7a4ea5","6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2108055343585038336","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-08T04:42:14Z","started_at":"2026-10-08T04:42:14Z","completed_at":"2026-10-08T07:16:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T07:20:00.897066011Z","timing_metadata_checked_at":"2026-10-08T07:20:00.897066011Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T07:20:00.911399464Z","region_metadata_checked_at":"2026-10-08T07:20:00.911399464Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2107988105985265664","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-08T00:15:03Z","started_at":"2026-10-08T00:15:03Z","completed_at":"2026-10-08T02:30:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T02:34:12.942663369Z","timing_metadata_checked_at":"2026-10-08T02:34:12.942663369Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T02:34:12.947766951Z","region_metadata_checked_at":"2026-10-08T02:34:12.947766951Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2bcee9e4c62bb8bf3094b01035a9d56fc62223c3e5ea60de974fe9f92ad2a4a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2107988105985265664","occurred_at":"2026-10-08T00:15:03Z","failed_at":"e2e","signature_id":"56eea831600730567aa0c1ce05d30226ca8a8d1e83b1c5e0f2bc75bde312d724","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2700c25b4198\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-1-7wd5gspw55gd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-wvvvhw\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-7wd5gspw55gd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd1fd723-1818-4cf3-abd1-cd25fc17377b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd1fd723-1818-4cf3-abd1-cd25fc17377b\",\n      \"name\": \"cd1fd723-1818-4cf3-abd1-cd25fc17377b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:41:07.332810869Z\",\n      \"endTime\": \"2026-10-08T02:05:10.961421785Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2taa15pl8rhqv7goa573mk2hebqeei6j still exists (deletion dispatched at 2026-10-08T01:41:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x2700c25b4198\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-1-7wd5gspw55gd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-1-wvvvhw\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-7wd5gspw55gd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd1fd723-1818-4cf3-abd1-cd25fc17377b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd1fd723-1818-4cf3-abd1-cd25fc17377b\",\n      \"name\": \"cd1fd723-1818-4cf3-abd1-cd25fc17377b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:41:07.332810869Z\",\n      \"endTime\": \"2026-10-08T02:05:10.961421785Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2taa15pl8rhqv7goa573mk2hebqeei6j still exists (deletion dispatched at 2026-10-08T01:41:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"60cbfcc093cc71a327f97e4555f0ac485556a218088fbd729807cbe318944606","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2107988105985265664","occurred_at":"2026-10-08T00:15:03Z","failed_at":"e2e","signature_id":"4687436a0bfcfca8d25b9c7ea62a3459ab48c2ddd70a922beb77b67c78af05b7","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x1a2d62cf4c18\u003e: \n    resource group \"node-limit-test-7ngnq4mvtzfd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-7ngnq4mvtzfd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7157c234-f4b7-4732-ad4a-89009867cbd7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7157c234-f4b7-4732-ad4a-89009867cbd7\",\n      \"name\": \"7157c234-f4b7-4732-ad4a-89009867cbd7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:35:15.90215739Z\",\n      \"endTime\": \"2026-10-08T01:59:21.185260922Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2taa7pjbbs0aafhd6esu75djenuq7kvc still exists (deletion dispatched at 2026-10-08T01:35:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x1a2d62cf4c18\u003e: \n    resource group \"node-limit-test-7ngnq4mvtzfd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-7ngnq4mvtzfd\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7157c234-f4b7-4732-ad4a-89009867cbd7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7157c234-f4b7-4732-ad4a-89009867cbd7\",\n      \"name\": \"7157c234-f4b7-4732-ad4a-89009867cbd7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:35:15.90215739Z\",\n      \"endTime\": \"2026-10-08T01:59:21.185260922Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2taa7pjbbs0aafhd6esu75djenuq7kvc still exists (deletion dispatched at 2026-10-08T01:35:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (1)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6d9fccc12fa8bda1231e531f3251d664803a5e3b3c6c7d054b84e6e900351ee0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}